diff --git a/api/v1/agent/client/daemonset/delete_ipam_ips_parameters.go b/api/v1/agent/client/daemonset/delete_ipam_ips_parameters.go index a50d3ee807..835b155f03 100644 --- a/api/v1/agent/client/daemonset/delete_ipam_ips_parameters.go +++ b/api/v1/agent/client/daemonset/delete_ipam_ips_parameters.go @@ -17,6 +17,8 @@ import ( "github.com/go-openapi/runtime" cr "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + + "github.com/spidernet-io/spiderpool/api/v1/agent/models" ) // NewDeleteIpamIpsParams creates a new DeleteIpamIpsParams object, @@ -63,6 +65,10 @@ DeleteIpamIpsParams contains all the parameters to send to the API endpoint Typically these are written to a http.Request. */ type DeleteIpamIpsParams struct { + + // IpamBatchDelArgs. + IpamBatchDelArgs *models.IpamBatchDelArgs + timeout time.Duration Context context.Context HTTPClient *http.Client @@ -116,6 +122,17 @@ func (o *DeleteIpamIpsParams) SetHTTPClient(client *http.Client) { o.HTTPClient = client } +// WithIpamBatchDelArgs adds the ipamBatchDelArgs to the delete ipam ips params +func (o *DeleteIpamIpsParams) WithIpamBatchDelArgs(ipamBatchDelArgs *models.IpamBatchDelArgs) *DeleteIpamIpsParams { + o.SetIpamBatchDelArgs(ipamBatchDelArgs) + return o +} + +// SetIpamBatchDelArgs adds the ipamBatchDelArgs to the delete ipam ips params +func (o *DeleteIpamIpsParams) SetIpamBatchDelArgs(ipamBatchDelArgs *models.IpamBatchDelArgs) { + o.IpamBatchDelArgs = ipamBatchDelArgs +} + // WriteToRequest writes these params to a swagger request func (o *DeleteIpamIpsParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { @@ -123,6 +140,11 @@ func (o *DeleteIpamIpsParams) WriteToRequest(r runtime.ClientRequest, reg strfmt return err } var res []error + if o.IpamBatchDelArgs != nil { + if err := r.SetBodyParam(o.IpamBatchDelArgs); err != nil { + return err + } + } if len(res) > 0 { return errors.CompositeValidationError(res...) diff --git a/api/v1/agent/client/daemonset/delete_ipam_ips_responses.go b/api/v1/agent/client/daemonset/delete_ipam_ips_responses.go index e1c259a918..da657a5bd5 100644 --- a/api/v1/agent/client/daemonset/delete_ipam_ips_responses.go +++ b/api/v1/agent/client/daemonset/delete_ipam_ips_responses.go @@ -38,6 +38,18 @@ func (o *DeleteIpamIpsReader) ReadResponse(response runtime.ClientResponse, cons return nil, err } return nil, result + case 521: + result := NewDeleteIpamIpsStatus521() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return nil, result + case 522: + result := NewDeleteIpamIpsStatus522() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return nil, result default: return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) } @@ -164,3 +176,135 @@ func (o *DeleteIpamIpsFailure) readResponse(response runtime.ClientResponse, con return nil } + +// NewDeleteIpamIpsStatus521 creates a DeleteIpamIpsStatus521 with default headers values +func NewDeleteIpamIpsStatus521() *DeleteIpamIpsStatus521 { + return &DeleteIpamIpsStatus521{} +} + +/* +DeleteIpamIpsStatus521 describes a response with status code 521, with default header values. + +Forbid to release IPs for stateless workload +*/ +type DeleteIpamIpsStatus521 struct { + Payload models.Error +} + +// IsSuccess returns true when this delete ipam ips status521 response has a 2xx status code +func (o *DeleteIpamIpsStatus521) IsSuccess() bool { + return false +} + +// IsRedirect returns true when this delete ipam ips status521 response has a 3xx status code +func (o *DeleteIpamIpsStatus521) IsRedirect() bool { + return false +} + +// IsClientError returns true when this delete ipam ips status521 response has a 4xx status code +func (o *DeleteIpamIpsStatus521) IsClientError() bool { + return false +} + +// IsServerError returns true when this delete ipam ips status521 response has a 5xx status code +func (o *DeleteIpamIpsStatus521) IsServerError() bool { + return true +} + +// IsCode returns true when this delete ipam ips status521 response a status code equal to that given +func (o *DeleteIpamIpsStatus521) IsCode(code int) bool { + return code == 521 +} + +// Code gets the status code for the delete ipam ips status521 response +func (o *DeleteIpamIpsStatus521) Code() int { + return 521 +} + +func (o *DeleteIpamIpsStatus521) Error() string { + return fmt.Sprintf("[DELETE /ipam/ips][%d] deleteIpamIpsStatus521 %+v", 521, o.Payload) +} + +func (o *DeleteIpamIpsStatus521) String() string { + return fmt.Sprintf("[DELETE /ipam/ips][%d] deleteIpamIpsStatus521 %+v", 521, o.Payload) +} + +func (o *DeleteIpamIpsStatus521) GetPayload() models.Error { + return o.Payload +} + +func (o *DeleteIpamIpsStatus521) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +// NewDeleteIpamIpsStatus522 creates a DeleteIpamIpsStatus522 with default headers values +func NewDeleteIpamIpsStatus522() *DeleteIpamIpsStatus522 { + return &DeleteIpamIpsStatus522{} +} + +/* +DeleteIpamIpsStatus522 describes a response with status code 522, with default header values. + +Forbid to release IPs for stateful workload +*/ +type DeleteIpamIpsStatus522 struct { + Payload models.Error +} + +// IsSuccess returns true when this delete ipam ips status522 response has a 2xx status code +func (o *DeleteIpamIpsStatus522) IsSuccess() bool { + return false +} + +// IsRedirect returns true when this delete ipam ips status522 response has a 3xx status code +func (o *DeleteIpamIpsStatus522) IsRedirect() bool { + return false +} + +// IsClientError returns true when this delete ipam ips status522 response has a 4xx status code +func (o *DeleteIpamIpsStatus522) IsClientError() bool { + return false +} + +// IsServerError returns true when this delete ipam ips status522 response has a 5xx status code +func (o *DeleteIpamIpsStatus522) IsServerError() bool { + return true +} + +// IsCode returns true when this delete ipam ips status522 response a status code equal to that given +func (o *DeleteIpamIpsStatus522) IsCode(code int) bool { + return code == 522 +} + +// Code gets the status code for the delete ipam ips status522 response +func (o *DeleteIpamIpsStatus522) Code() int { + return 522 +} + +func (o *DeleteIpamIpsStatus522) Error() string { + return fmt.Sprintf("[DELETE /ipam/ips][%d] deleteIpamIpsStatus522 %+v", 522, o.Payload) +} + +func (o *DeleteIpamIpsStatus522) String() string { + return fmt.Sprintf("[DELETE /ipam/ips][%d] deleteIpamIpsStatus522 %+v", 522, o.Payload) +} + +func (o *DeleteIpamIpsStatus522) GetPayload() models.Error { + return o.Payload +} + +func (o *DeleteIpamIpsStatus522) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} diff --git a/api/v1/agent/models/ipam_batch_del_args.go b/api/v1/agent/models/ipam_batch_del_args.go new file mode 100644 index 0000000000..c6a7676c51 --- /dev/null +++ b/api/v1/agent/models/ipam_batch_del_args.go @@ -0,0 +1,131 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// IpamBatchDelArgs IPAM release IPs information +// +// swagger:model IpamBatchDelArgs +type IpamBatchDelArgs struct { + + // container ID + // Required: true + ContainerID *string `json:"containerID"` + + // is release conflict i ps + IsReleaseConflictIPs bool `json:"isReleaseConflictIPs,omitempty"` + + // net namespace + NetNamespace string `json:"netNamespace,omitempty"` + + // pod name + // Required: true + PodName *string `json:"podName"` + + // pod namespace + // Required: true + PodNamespace *string `json:"podNamespace"` + + // pod UID + // Required: true + PodUID *string `json:"podUID"` +} + +// Validate validates this ipam batch del args +func (m *IpamBatchDelArgs) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateContainerID(formats); err != nil { + res = append(res, err) + } + + if err := m.validatePodName(formats); err != nil { + res = append(res, err) + } + + if err := m.validatePodNamespace(formats); err != nil { + res = append(res, err) + } + + if err := m.validatePodUID(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *IpamBatchDelArgs) validateContainerID(formats strfmt.Registry) error { + + if err := validate.Required("containerID", "body", m.ContainerID); err != nil { + return err + } + + return nil +} + +func (m *IpamBatchDelArgs) validatePodName(formats strfmt.Registry) error { + + if err := validate.Required("podName", "body", m.PodName); err != nil { + return err + } + + return nil +} + +func (m *IpamBatchDelArgs) validatePodNamespace(formats strfmt.Registry) error { + + if err := validate.Required("podNamespace", "body", m.PodNamespace); err != nil { + return err + } + + return nil +} + +func (m *IpamBatchDelArgs) validatePodUID(formats strfmt.Registry) error { + + if err := validate.Required("podUID", "body", m.PodUID); err != nil { + return err + } + + return nil +} + +// ContextValidate validates this ipam batch del args based on context it is used +func (m *IpamBatchDelArgs) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *IpamBatchDelArgs) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *IpamBatchDelArgs) UnmarshalBinary(b []byte) error { + var res IpamBatchDelArgs + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/api/v1/agent/openapi.yaml b/api/v1/agent/openapi.yaml index 30976321b9..837f207413 100644 --- a/api/v1/agent/openapi.yaml +++ b/api/v1/agent/openapi.yaml @@ -88,6 +88,12 @@ paths: Delete multiple ip for a pod, case for spiderflat compent tags: - daemonset + parameters: + - name: ipam-batch-del-args + in: body + required: true + schema: + $ref: "#/definitions/IpamBatchDelArgs" responses: "200": description: Success @@ -96,6 +102,14 @@ paths: x-go-name: Failure schema: $ref: "#/definitions/Error" + '521': + description: Forbid to release IPs for stateless workload + schema: + $ref: "#/definitions/Error" + '522': + description: Forbid to release IPs for stateful workload + schema: + $ref: "#/definitions/Error" "/workloadendpoint": get: summary: Get workloadendpoint status @@ -347,3 +361,24 @@ definitions: type: string podNamespace: type: string + IpamBatchDelArgs: + description: IPAM release IPs information + type: object + properties: + isReleaseConflictIPs: + type: boolean + containerID: + type: string + netNamespace: + type: string + podNamespace: + type: string + podName: + type: string + podUID: + type: string + required: + - containerID + - podNamespace + - podName + - podUID diff --git a/api/v1/agent/server/embedded_spec.go b/api/v1/agent/server/embedded_spec.go index 7f2e22ae1e..11b21cbd64 100644 --- a/api/v1/agent/server/embedded_spec.go +++ b/api/v1/agent/server/embedded_spec.go @@ -175,6 +175,16 @@ func init() { "daemonset" ], "summary": "Delete multiple ip as a batch", + "parameters": [ + { + "name": "ipam-batch-del-args", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/IpamBatchDelArgs" + } + } + ], "responses": { "200": { "description": "Success" @@ -185,6 +195,18 @@ func init() { "$ref": "#/definitions/Error" }, "x-go-name": "Failure" + }, + "521": { + "description": "Forbid to release IPs for stateless workload", + "schema": { + "$ref": "#/definitions/Error" + } + }, + "522": { + "description": "Forbid to release IPs for stateful workload", + "schema": { + "$ref": "#/definitions/Error" + } } } } @@ -470,6 +492,36 @@ func init() { } } }, + "IpamBatchDelArgs": { + "description": "IPAM release IPs information", + "type": "object", + "required": [ + "containerID", + "podNamespace", + "podName", + "podUID" + ], + "properties": { + "containerID": { + "type": "string" + }, + "isReleaseConflictIPs": { + "type": "boolean" + }, + "netNamespace": { + "type": "string" + }, + "podName": { + "type": "string" + }, + "podNamespace": { + "type": "string" + }, + "podUID": { + "type": "string" + } + } + }, "IpamDelArgs": { "description": "IPAM release IP information", "type": "object", @@ -681,6 +733,16 @@ func init() { "daemonset" ], "summary": "Delete multiple ip as a batch", + "parameters": [ + { + "name": "ipam-batch-del-args", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/IpamBatchDelArgs" + } + } + ], "responses": { "200": { "description": "Success" @@ -691,6 +753,18 @@ func init() { "$ref": "#/definitions/Error" }, "x-go-name": "Failure" + }, + "521": { + "description": "Forbid to release IPs for stateless workload", + "schema": { + "$ref": "#/definitions/Error" + } + }, + "522": { + "description": "Forbid to release IPs for stateful workload", + "schema": { + "$ref": "#/definitions/Error" + } } } } @@ -976,6 +1050,36 @@ func init() { } } }, + "IpamBatchDelArgs": { + "description": "IPAM release IPs information", + "type": "object", + "required": [ + "containerID", + "podNamespace", + "podName", + "podUID" + ], + "properties": { + "containerID": { + "type": "string" + }, + "isReleaseConflictIPs": { + "type": "boolean" + }, + "netNamespace": { + "type": "string" + }, + "podName": { + "type": "string" + }, + "podNamespace": { + "type": "string" + }, + "podUID": { + "type": "string" + } + } + }, "IpamDelArgs": { "description": "IPAM release IP information", "type": "object", diff --git a/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_parameters.go b/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_parameters.go index 4009f2cae9..5fefab4375 100644 --- a/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_parameters.go +++ b/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_parameters.go @@ -9,10 +9,15 @@ package daemonset // Editing this file might prove futile when you re-run the swagger generate command import ( + "io" "net/http" "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/validate" + + "github.com/spidernet-io/spiderpool/api/v1/agent/models" ) // NewDeleteIpamIpsParams creates a new DeleteIpamIpsParams object @@ -31,6 +36,12 @@ type DeleteIpamIpsParams struct { // HTTP Request Object HTTPRequest *http.Request `json:"-"` + + /* + Required: true + In: body + */ + IpamBatchDelArgs *models.IpamBatchDelArgs } // BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface @@ -42,6 +53,33 @@ func (o *DeleteIpamIpsParams) BindRequest(r *http.Request, route *middleware.Mat o.HTTPRequest = r + if runtime.HasBody(r) { + defer r.Body.Close() + var body models.IpamBatchDelArgs + if err := route.Consumer.Consume(r.Body, &body); err != nil { + if err == io.EOF { + res = append(res, errors.Required("ipamBatchDelArgs", "body", "")) + } else { + res = append(res, errors.NewParseError("ipamBatchDelArgs", "body", "", err)) + } + } else { + // validate body object + if err := body.Validate(route.Formats); err != nil { + res = append(res, err) + } + + ctx := validate.WithOperationRequest(r.Context()) + if err := body.ContextValidate(ctx, route.Formats); err != nil { + res = append(res, err) + } + + if len(res) == 0 { + o.IpamBatchDelArgs = &body + } + } + } else { + res = append(res, errors.Required("ipamBatchDelArgs", "body", "")) + } if len(res) > 0 { return errors.CompositeValidationError(res...) } diff --git a/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_responses.go b/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_responses.go index 69689b48ab..de6ff855e4 100644 --- a/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_responses.go +++ b/api/v1/agent/server/restapi/daemonset/delete_ipam_ips_responses.go @@ -83,3 +83,89 @@ func (o *DeleteIpamIpsFailure) WriteResponse(rw http.ResponseWriter, producer ru panic(err) // let the recovery middleware deal with this } } + +// DeleteIpamIpsStatus521Code is the HTTP code returned for type DeleteIpamIpsStatus521 +const DeleteIpamIpsStatus521Code int = 521 + +/* +DeleteIpamIpsStatus521 Forbid to release IPs for stateless workload + +swagger:response deleteIpamIpsStatus521 +*/ +type DeleteIpamIpsStatus521 struct { + + /* + In: Body + */ + Payload models.Error `json:"body,omitempty"` +} + +// NewDeleteIpamIpsStatus521 creates DeleteIpamIpsStatus521 with default headers values +func NewDeleteIpamIpsStatus521() *DeleteIpamIpsStatus521 { + + return &DeleteIpamIpsStatus521{} +} + +// WithPayload adds the payload to the delete ipam ips status521 response +func (o *DeleteIpamIpsStatus521) WithPayload(payload models.Error) *DeleteIpamIpsStatus521 { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the delete ipam ips status521 response +func (o *DeleteIpamIpsStatus521) SetPayload(payload models.Error) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *DeleteIpamIpsStatus521) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(521) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + +// DeleteIpamIpsStatus522Code is the HTTP code returned for type DeleteIpamIpsStatus522 +const DeleteIpamIpsStatus522Code int = 522 + +/* +DeleteIpamIpsStatus522 Forbid to release IPs for stateful workload + +swagger:response deleteIpamIpsStatus522 +*/ +type DeleteIpamIpsStatus522 struct { + + /* + In: Body + */ + Payload models.Error `json:"body,omitempty"` +} + +// NewDeleteIpamIpsStatus522 creates DeleteIpamIpsStatus522 with default headers values +func NewDeleteIpamIpsStatus522() *DeleteIpamIpsStatus522 { + + return &DeleteIpamIpsStatus522{} +} + +// WithPayload adds the payload to the delete ipam ips status522 response +func (o *DeleteIpamIpsStatus522) WithPayload(payload models.Error) *DeleteIpamIpsStatus522 { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the delete ipam ips status522 response +func (o *DeleteIpamIpsStatus522) SetPayload(payload models.Error) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *DeleteIpamIpsStatus522) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(522) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spiderendpoints.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spiderendpoints.yaml index df5028edc9..28ba9b1a33 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spiderendpoints.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spiderendpoints.yaml @@ -98,6 +98,7 @@ spec: type: array vlan: default: 0 + description: 'DEPRECATED: Vlan is deprecated.' format: int64 maximum: 4094 minimum: 0 diff --git a/cmd/coordinator/cmd/command_add.go b/cmd/coordinator/cmd/command_add.go index 6d4e745423..5e9d7307ab 100644 --- a/cmd/coordinator/cmd/command_add.go +++ b/cmd/coordinator/cmd/command_add.go @@ -4,6 +4,8 @@ package cmd import ( + "context" + "errors" "fmt" "time" @@ -11,14 +13,15 @@ import ( "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ns" - "github.com/spidernet-io/spiderpool/pkg/errgroup" "github.com/vishvananda/netlink" + "go.uber.org/multierr" "go.uber.org/zap" "github.com/spidernet-io/spiderpool/api/v1/agent/client/daemonset" "github.com/spidernet-io/spiderpool/api/v1/agent/models" plugincmd "github.com/spidernet-io/spiderpool/cmd/spiderpool/cmd" "github.com/spidernet-io/spiderpool/pkg/constant" + "github.com/spidernet-io/spiderpool/pkg/errgroup" "github.com/spidernet-io/spiderpool/pkg/logutils" "github.com/spidernet-io/spiderpool/pkg/networking/gwconnection" "github.com/spidernet-io/spiderpool/pkg/networking/ipchecking" @@ -184,9 +187,10 @@ func CmdAdd(args *skel.CmdArgs) (err error) { logger.Debug("disable detect gateway") } + var ipc *ipchecking.IPChecker if conf.IPConflict != nil && *conf.IPConflict { logger.Debug("Try to detect ip conflict") - ipc, err := ipchecking.NewIPChecker(conf.DetectOptions.Retry, conf.DetectOptions.Interval, conf.DetectOptions.TimeOut, c.hostNs, c.netns, logger) + ipc, err = ipchecking.NewIPChecker(conf.DetectOptions.Retry, conf.DetectOptions.Interval, conf.DetectOptions.TimeOut, c.hostNs, c.netns, logger) if err != nil { return fmt.Errorf("failed to run NewIPChecker: %w", err) } @@ -197,6 +201,22 @@ func CmdAdd(args *skel.CmdArgs) (err error) { if err = errg.Wait(); err != nil { logger.Error("failed to detect gateway and ip checking", zap.Error(err)) + if errors.Is(err, constant.ErrIPConflict) { + _, innerErr := client.Daemonset.DeleteIpamIps(daemonset.NewDeleteIpamIpsParams().WithContext(context.TODO()).WithIpamBatchDelArgs( + &models.IpamBatchDelArgs{ + ContainerID: &args.ContainerID, + NetNamespace: args.Netns, + PodName: (*string)(&k8sArgs.K8S_POD_NAME), + PodNamespace: (*string)(&k8sArgs.K8S_POD_NAMESPACE), + PodUID: (*string)(&k8sArgs.K8S_POD_UID), + }, + )) + if nil != innerErr { + logger.Sugar().Errorf("failed to clean up conflict IPs, error: %v", innerErr) + return multierr.Append(err, innerErr) + } + } + return err } diff --git a/cmd/spiderpool-agent/cmd/config.go b/cmd/spiderpool-agent/cmd/config.go index 3741e85f5d..90d71c470f 100644 --- a/cmd/spiderpool-agent/cmd/config.go +++ b/cmd/spiderpool-agent/cmd/config.go @@ -59,6 +59,7 @@ var envInfo = []envConf{ {"SPIDERPOOL_METRIC_HTTP_PORT", "5711", true, &agentContext.Cfg.MetricHttpPort, nil, nil}, {"SPIDERPOOL_GOPS_LISTEN_PORT", "5712", false, &agentContext.Cfg.GopsListenPort, nil, nil}, {"SPIDERPOOL_PYROSCOPE_PUSH_SERVER_ADDRESS", "", false, &agentContext.Cfg.PyroscopeAddress, nil, nil}, + {"SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS", "true", true, nil, &agentContext.Cfg.EnableReleaseConflictIPsForStateless, nil}, {"SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS", "5000", true, nil, nil, &agentContext.Cfg.IPPoolMaxAllocatedIPs}, {"SPIDERPOOL_WAIT_SUBNET_POOL_TIME_IN_SECOND", "2", false, nil, nil, &agentContext.Cfg.WaitSubnetPoolTime}, @@ -77,11 +78,12 @@ type Config struct { ConfigPath string // env - LogLevel string - EnableMetric bool - EnableDebugLevelMetric bool - AgentPodNamespace string - AgentPodName string + LogLevel string + EnableMetric bool + EnableDebugLevelMetric bool + AgentPodNamespace string + AgentPodName string + EnableReleaseConflictIPsForStateless bool HttpPort string MetricHttpPort string diff --git a/cmd/spiderpool-agent/cmd/daemon.go b/cmd/spiderpool-agent/cmd/daemon.go index 25e75ca88f..1b197ceab4 100644 --- a/cmd/spiderpool-agent/cmd/daemon.go +++ b/cmd/spiderpool-agent/cmd/daemon.go @@ -132,14 +132,15 @@ func DaemonMain() { logger.Info("Begin to initialize IPAM") ipamConfig := ipam.IPAMConfig{ - EnableIPv4: agentContext.Cfg.EnableIPv4, - EnableIPv6: agentContext.Cfg.EnableIPv6, - EnableSpiderSubnet: agentContext.Cfg.EnableSpiderSubnet, - EnableStatefulSet: agentContext.Cfg.EnableStatefulSet, - EnableKubevirtStaticIP: agentContext.Cfg.EnableKubevirtStaticIP, - OperationRetries: agentContext.Cfg.WaitSubnetPoolMaxRetries, - OperationGapDuration: time.Duration(agentContext.Cfg.WaitSubnetPoolTime) * time.Second, - AgentNamespace: agentContext.Cfg.AgentPodNamespace, + EnableIPv4: agentContext.Cfg.EnableIPv4, + EnableIPv6: agentContext.Cfg.EnableIPv6, + EnableSpiderSubnet: agentContext.Cfg.EnableSpiderSubnet, + EnableStatefulSet: agentContext.Cfg.EnableStatefulSet, + EnableKubevirtStaticIP: agentContext.Cfg.EnableKubevirtStaticIP, + EnableReleaseConflictIPsForStateless: agentContext.Cfg.EnableReleaseConflictIPsForStateless, + OperationRetries: agentContext.Cfg.WaitSubnetPoolMaxRetries, + OperationGapDuration: time.Duration(agentContext.Cfg.WaitSubnetPoolTime) * time.Second, + AgentNamespace: agentContext.Cfg.AgentPodNamespace, } if len(agentContext.Cfg.MultusClusterNetwork) != 0 { ipamConfig.MultusClusterNetwork = ptr.To(agentContext.Cfg.MultusClusterNetwork) diff --git a/cmd/spiderpool-agent/cmd/ipam.go b/cmd/spiderpool-agent/cmd/ipam.go index 150f5292dc..21cadca475 100644 --- a/cmd/spiderpool-agent/cmd/ipam.go +++ b/cmd/spiderpool-agent/cmd/ipam.go @@ -74,7 +74,7 @@ type _unixDeleteAgentIpamIp struct{} // Handle handles DELETE requests for /ipam/ip. func (g *_unixDeleteAgentIpamIp) Handle(params daemonset.DeleteIpamIPParams) middleware.Responder { if err := params.IpamDelArgs.Validate(strfmt.Default); err != nil { - return daemonset.NewPostIpamIPFailure().WithPayload(models.Error(err.Error())) + return daemonset.NewDeleteIpamIPFailure().WithPayload(models.Error(err.Error())) } logger := logutils.Logger.Named("IPAM").With( @@ -122,6 +122,41 @@ type _unixDeleteAgentIpamIps struct{} // Handle handles DELETE requests for /ipam/ips. func (g *_unixDeleteAgentIpamIps) Handle(params daemonset.DeleteIpamIpsParams) middleware.Responder { + err := params.IpamBatchDelArgs.Validate(strfmt.Default) + if nil != err { + return daemonset.NewDeleteIpamIpsFailure().WithPayload(models.Error(err.Error())) + } + + log := logutils.Logger.Named("IPAM").With( + zap.String("Operation", "Release IPs"), + zap.String("ContainerID", *params.IpamBatchDelArgs.ContainerID), + zap.String("NetNamespace", params.IpamBatchDelArgs.NetNamespace), + zap.String("PodNamespace", *params.IpamBatchDelArgs.PodNamespace), + zap.String("PodName", *params.IpamBatchDelArgs.PodName), + zap.String("PodUID", *params.IpamBatchDelArgs.PodUID), + ) + ctx := logutils.IntoContext(params.HTTPRequest.Context(), log) + + // The total count of IP releasing. + metric.IpamReleaseTotalCounts.Add(ctx, 1) + + timeRecorder := metric.NewTimeRecorder() + defer func() { + // Time taken for once IP releasing. + releaseDuration := timeRecorder.SinceInSeconds() + metric.IPAMDurationConstruct.RecordIPAMReleaseDuration(ctx, releaseDuration) + logger.Sugar().Infof("IPAM releasing duration: %v", releaseDuration) + }() + + err = agentContext.IPAM.ReleaseIPs(ctx, params.IpamBatchDelArgs) + if nil != err { + // The count of failures in IP releasing. + metric.IpamReleaseFailureCounts.Add(ctx, 1) + gatherIPAMReleasingErrMetric(ctx, err) + logger.Error(err.Error()) + return filteredErrResponder(err) + } + return daemonset.NewDeleteIpamIpsOK() } @@ -160,3 +195,14 @@ func gatherIPAMReleasingErrMetric(ctx context.Context, err error) { metric.IpamReleaseErrInternalCounts.Add(ctx, 1) } } + +func filteredErrResponder(err error) middleware.Responder { + switch { + case errors.Is(err, constant.ErrForbidReleasingStatelessWorkload): + return daemonset.NewDeleteIpamIpsStatus521().WithPayload(models.Error(err.Error())) + case errors.Is(err, constant.ErrForbidReleasingStatefulWorkload): + return daemonset.NewDeleteIpamIpsStatus522().WithPayload(models.Error(err.Error())) + default: + return daemonset.NewDeleteIpamIpsFailure().WithPayload(models.Error(err.Error())) + } +} diff --git a/docs/concepts/coordinator-zh_CN.md b/docs/concepts/coordinator-zh_CN.md index f084160be7..b5dfe911e1 100644 --- a/docs/concepts/coordinator-zh_CN.md +++ b/docs/concepts/coordinator-zh_CN.md @@ -1,6 +1,6 @@ # Coordinator -[**English**](coordinator.md) | **简体中文** +**简体中文** | [**English**](coordinator.md) Spiderpool 内置一个叫 `coordinator` 的 CNI meta-plugin, 它在 Main CNI 被调用之后再工作，它主要提供以下几个主要功能: @@ -57,7 +57,8 @@ ClusterIP 的路由，导致无法访问。 ## 支持检测 Pod 的 IP 是否冲突( alpha 阶段) 对于 Underlay 网络，IP 冲突是无法接受的，这可能会造成严重的问题。在创建 Pod 时，我们可借助 `coordinator` 检测 Pod 的 IP 是否冲突，支持同时检测 IPv4 和 IPv6 地址。通过发送 ARP 或 NDP 探测报文， -如果发现回复报文的 Mac 地址不是 Pod 本身，那我们认为这个 IP 是冲突的，并拒绝 IP 冲突的 Pod 被创建: +如果发现回复报文的 Mac 地址不是来自 Pod 本身的网卡，那我们认为这个 IP 是冲突的，并拒绝 IP 冲突的 Pod 被创建。 +此外，我们默认还会对发生 IP 冲突的**无状态**的 Pod 释放所有的已分配的 IP 使其重新分配，使得 Pod 在下一次重新调用 CNI 时能够尝试分配到其它非冲突的 IP。对于发生 IP 冲突的**有状态**的 Pod，为了保持 IP 地址也是有状态设计，我们不会对其释放。您可通过 spiderpool-agent [环境变量](../reference/spiderpool-agent.md#env) `SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS` 来控制此功能。 我们可以通过 Spidermultusconfig 配置它: diff --git a/docs/concepts/coordinator.md b/docs/concepts/coordinator.md index e2e588c3e8..fc6a251c6c 100644 --- a/docs/concepts/coordinator.md +++ b/docs/concepts/coordinator.md @@ -57,7 +57,8 @@ For more information about the Underlay Pod not being able to access the Cluster ## Detect Pod IP conflicts(alpha) IP conflicts are unacceptable for underlay networks, which can cause serious problems. When creating a pod, we can use the `coordinator` to detect whether the IP of the pod conflicts, and support both IPv4 and IPv6 addresses. By sending an ARP or NDP probe message, -If the MAC address of the reply packet is not the pod itself, we consider the IP to be conflicting and reject the creation of the pod with conflicting IP addresses: +If the MAC address of the reply packet does not belong to the Pod NIC, we consider the IP to be in conflict and reject the creation of the pod with conflicting IP addresses. +Additionally, we will default to release the whole allocated IPs for the **stateless** Pod to make it try to reallocate those no-conflict IPs in the next CNI call for the Pod. For the **stable** Pod with conflict IPs, we would not release its IPs to keep the IPs own the stable feature either. You can use spiderpool-agent [ENV](../reference/spiderpool-agent.md#env) `SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS` to control this feature. ```yaml apiVersion: spiderpool.spidernet.io/v2beta1 diff --git a/docs/develop/roadmap.md b/docs/develop/roadmap.md index 5ff38b51f9..e608b1cc70 100644 --- a/docs/develop/roadmap.md +++ b/docs/develop/roadmap.md @@ -1,70 +1,71 @@ # roadmap -| feature | description | Alpha release | Beta release | GA release | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------------|---------------|--------------|------------| -| SpiderIppool | ip settings | v0.2.0 | v0.4.0 | v0.6.0 | -| | namespace affinity | v0.4.0 | v0.6.0 | | -| | application affinity | v0.4.0 | v0.6.0 | | -| | multiple default ippool | v0.6.0 | | | -| | multusname affinity | v0.6.0 | | | -| | nodename affinity | v0.6.0 | v0.6.0 | -| | default cluster ippool | v0.2.0 | v0.4.0 | v0.6.0 | -| | default namespace ippool | v0.4.0 | v0.5.0 | | -| | default CNI ippool | v0.4.0 | v0.4.0 | | -| | annotation ippool | v0.2.0 | v0.5.0 | | -| | annotation route | v0.2.0 | v0.5.0 | | -| | ippools for multi-interfaces without specified interface name in annotation | v0.9.0 | | | -| SpiderSubnet | automatically create ippool | v0.4.0 | | | -| | automatically scaling and deletion ip according to application | v0.4.0 | | | -| | automatically delete ippool | v0.5.0 | | | -| | annotation for multiple interface | v0.4.0 | | | -| | keep ippool after deleting application | v0.5.0 | | | -| | support deployment, statefulset, job, replicaset | v0.4.0 | | | -| | support operator controller | v0.4.0 | | | -| | flexible ip number | v0.5.0 | | | -| | ippool inherit route and gateway attribute from its subnet | v0.6.0 | | | -| reservedIP | reservedIP | v0.4.0 | v0.6.0 | | -| Fixed IP | fixed ip for each pod of statefulset | v0.5.0 | | | -| | fixed ip ranges for statefulset, deployment, replicaset | v0.4.0 | v0.6.0 | | -| | fixed ip for kubevirt | v0.8.0 | | | -| | support calico | v0.5.0 | v0.6.0 | | -| | support weave | v0.5.0 | v0.6.0 | | -| Spidermultusconfig | support macvlan ipvlan sriov custom | v0.6.0 | v0.7.0 | | -| | support ovs-cni | v0.7.0 | | | -| CNI version | cni v1.0.0 | v0.4.0 | v0.5.0 | | -| ifacer | bond interface | v0.6.0 | v0.8.0 | | -| | vlan interface | v0.6.0 | v0.8.0 | | -| SpiderCoordinator | Sync podCIDR for calico | v0.6.0 | v0.8.0 -| | Sync podCIDR for cilium | v0.6.0 | v0.8.0 -| | sync clusterIP CIDR from serviceCIDR to support k8s 1.29 | | v0.10.0 | -| Coordinator | support underlay mode | v0.6.0 | v0.7.0 | | -| | support overlay mode | v0.6.0 | v0.8.0 | | -| | CRD spidercoordinators for multus configuration | v0.6.0 | v0.8.0 | | -| | detect ip conflict and gateway | v0.6.0 | v0.6.0 | | -| | specify the MAC of pod | v0.6.0 | v0.8.0 | | -| | tune the default route of pod multiple interfaces | v0.6.0 | v0.8.0 | | -| Connectivity | visit service based on kube-proxy | v0.6.0 | v0.7.0 | | -| | visit local node to guarantee the pod health check | v0.6.0 | v0.7.0 | | -| | visit nodePort with spec.externalTrafficPolicy=local or spec.externalTrafficPolicy=cluster | v0.6.0 | | | -| Observability | eBPF: pod stats | In plan | | | -| Network Policy | ipvlan | v0.8.0 | | | -| | macvlan | In plan | | | -| | sriov | In plan | | | -| Bandwidth | ipvlan | v0.8.0 | | | -| | macvlan | In plan | | | -| | sriov | In plan | | | -| eBPF | implement service by cgroup eBPF | v0.8.0 | | | -| | accelerate communication of pods on a same node | In plan | | | -| Recycle IP | recycle IP taken by deleted pod | v0.4.0 | v0.6.0 | | -| | recycle IP taken by deleting pod | v0.4.0 | v0.6.0 | | -| Dual Stack | dual-stack | v0.2.0 | v0.4.0 | | -| CLI | debug and operate. check which pod an IP is taken by, check IP usage , trigger GC | In plan | | | -| Multi-cluster | a broker cluster could synchronize ippool resource within a same subnet from all member clusters, which could help avoid IP conflict | In plan | | | -| | support submariner | v0.8.0 | | | -| Dual CNI | underlay cooperate with cilium | v0.7.0 | | | -| | underlay cooperate with calico | v0.7.0 | | | -| RDMA | support macvlan and ipvlan CNI for RoCE device | v0.8.0 | | | -| | support sriov CNI for RoCE device | v0.8.0 | | | -| | support ipoib CNI for infiniband device | v0.9.0 | | | -| | support ib-sriov CNI for infiniband device | v0.9.0 | | | -| EgressGateway | egressGateway | v0.8.0 | | | +| feature | description | Alpha release | Beta release | GA release | +|--------------------|--------------------------------------------------------------------------------------------------------------------------------------|---------------|--------------|------------| +| SpiderIppool | ip settings | v0.2.0 | v0.4.0 | v0.6.0 | +| | namespace affinity | v0.4.0 | v0.6.0 | | +| | application affinity | v0.4.0 | v0.6.0 | | +| | multiple default ippool | v0.6.0 | | | +| | multusname affinity | v0.6.0 | | | +| | nodename affinity | v0.6.0 | v0.6.0 | | +| | default cluster ippool | v0.2.0 | v0.4.0 | v0.6.0 | +| | default namespace ippool | v0.4.0 | v0.5.0 | | +| | default CNI ippool | v0.4.0 | v0.4.0 | | +| | annotation ippool | v0.2.0 | v0.5.0 | | +| | annotation route | v0.2.0 | v0.5.0 | | +| | ippools for multi-interfaces without specified interface name in annotation | v0.9.0 | | | +| SpiderSubnet | automatically create ippool | v0.4.0 | | | +| | automatically scaling and deletion ip according to application | v0.4.0 | | | +| | automatically delete ippool | v0.5.0 | | | +| | annotation for multiple interface | v0.4.0 | | | +| | keep ippool after deleting application | v0.5.0 | | | +| | support deployment, statefulset, job, replicaset | v0.4.0 | | | +| | support operator controller | v0.4.0 | | | +| | flexible ip number | v0.5.0 | | | +| | ippool inherit route and gateway attribute from its subnet | v0.6.0 | | | +| reservedIP | reservedIP | v0.4.0 | v0.6.0 | | +| Fixed IP | fixed ip for each pod of statefulset | v0.5.0 | | | +| | fixed ip ranges for statefulset, deployment, replicaset | v0.4.0 | v0.6.0 | | +| | fixed ip for kubevirt | v0.8.0 | | | +| | support calico | v0.5.0 | v0.6.0 | | +| | support weave | v0.5.0 | v0.6.0 | | +| Spidermultusconfig | support macvlan ipvlan sriov custom | v0.6.0 | v0.7.0 | | +| | support ovs-cni | v0.7.0 | | | +| CNI version | cni v1.0.0 | v0.4.0 | v0.5.0 | | +| ifacer | bond interface | v0.6.0 | v0.8.0 | | +| | vlan interface | v0.6.0 | v0.8.0 | | +| SpiderCoordinator | Sync podCIDR for calico | v0.6.0 | v0.8.0 | | +| | Sync podCIDR for cilium | v0.6.0 | v0.8.0 | | +| | sync clusterIP CIDR from serviceCIDR to support k8s 1.29 | | v0.10.0 | | +| Coordinator | support underlay mode | v0.6.0 | v0.7.0 | | +| | support overlay mode | v0.6.0 | v0.8.0 | | +| | CRD spidercoordinators for multus configuration | v0.6.0 | v0.8.0 | | +| | detect ip conflict and gateway | v0.6.0 | v0.6.0 | | +| | specify the MAC of pod | v0.6.0 | v0.8.0 | | +| | tune the default route of pod multiple interfaces | v0.6.0 | v0.8.0 | | +| Connectivity | visit service based on kube-proxy | v0.6.0 | v0.7.0 | | +| | visit local node to guarantee the pod health check | v0.6.0 | v0.7.0 | | +| | visit nodePort with spec.externalTrafficPolicy=local or spec.externalTrafficPolicy=cluster | v0.6.0 | | | +| Observability | eBPF: pod stats | In plan | | | +| Network Policy | ipvlan | v0.8.0 | | | +| | macvlan | In plan | | | +| | sriov | In plan | | | +| Bandwidth | ipvlan | v0.8.0 | | | +| | macvlan | In plan | | | +| | sriov | In plan | | | +| eBPF | implement service by cgroup eBPF | v0.8.0 | | | +| | accelerate communication of pods on a same node | In plan | | | +| Recycle IP | recycle IP taken by deleted pod | v0.4.0 | v0.6.0 | | +| | recycle IP taken by deleting pod | v0.4.0 | v0.6.0 | | +| | recycle IP when detected IP conflict | v0.10.0 | | | +| Dual Stack | dual-stack | v0.2.0 | v0.4.0 | | +| CLI | debug and operate. check which pod an IP is taken by, check IP usage , trigger GC | In plan | | | +| Multi-cluster | a broker cluster could synchronize ippool resource within a same subnet from all member clusters, which could help avoid IP conflict | In plan | | | +| | support submariner | v0.8.0 | | | +| Dual CNI | underlay cooperate with cilium | v0.7.0 | | | +| | underlay cooperate with calico | v0.7.0 | | | +| RDMA | support macvlan and ipvlan CNI for RoCE device | v0.8.0 | | | +| | support sriov CNI for RoCE device | v0.8.0 | | | +| | support ipoib CNI for infiniband device | v0.9.0 | | | +| | support ib-sriov CNI for infiniband device | v0.9.0 | | | +| EgressGateway | egressGateway | v0.8.0 | | | diff --git a/docs/reference/spiderpool-agent.md b/docs/reference/spiderpool-agent.md index af9bfb848d..8f44b07881 100644 --- a/docs/reference/spiderpool-agent.md +++ b/docs/reference/spiderpool-agent.md @@ -16,7 +16,7 @@ Run the spiderpool agent daemon. ### ENV | env | default | description | -| ----------------------------------------------- | ------- |-------------------------------------------------------------------------------------------------| +|-------------------------------------------------|---------|-------------------------------------------------------------------------------------------------| | SPIDERPOOL_LOG_LEVEL | info | Log level, optional values are "debug", "info", "warn", "error", "fatal", "panic". | | SPIDERPOOL_ENABLED_METRIC | false | Enable/disable metrics. | | SPIDERPOOL_HEALTH_PORT | 5710 | Metric HTTP server port. | @@ -25,6 +25,7 @@ Run the spiderpool agent daemon. | SPIDERPOOL_UPDATE_CR_MAX_RETRIES | 3 | Max retries to update k8s resources. | | SPIDERPOOL_WORKLOADENDPOINT_MAX_HISTORY_RECORDS | 100 | Max historical IP allocation information allowed for a single Pod recorded in WorkloadEndpoint. | | SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS | 5000 | Max number of IP that a single IP pool can provide. | +| SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS | true | Enable/disable release conflict IPs. | ## spiderpool-agent shutdown diff --git a/pkg/constant/errors.go b/pkg/constant/errors.go index f08bf50407..2cbe603e44 100644 --- a/pkg/constant/errors.go +++ b/pkg/constant/errors.go @@ -8,10 +8,13 @@ import ( ) var ( - ErrWrongInput = errors.New("wrong input") - ErrNoAvailablePool = errors.New("no IPPool available") - ErrRetriesExhausted = errors.New("exhaust all retries") - ErrIPUsedOut = errors.New("all IP addresses used out") + ErrWrongInput = errors.New("wrong input") + ErrNoAvailablePool = errors.New("no IPPool available") + ErrRetriesExhausted = errors.New("exhaust all retries") + ErrIPUsedOut = errors.New("all IP addresses used out") + ErrIPConflict = errors.New("ip conflict") + ErrForbidReleasingStatefulWorkload = errors.New("forbid releasing IPs for stateful workload ") + ErrForbidReleasingStatelessWorkload = errors.New("forbid releasing IPs for stateless workload") ) var ErrMissingRequiredParam = errors.New("must be specified") diff --git a/pkg/ipam/config.go b/pkg/ipam/config.go index 4f02037134..02d950589c 100644 --- a/pkg/ipam/config.go +++ b/pkg/ipam/config.go @@ -18,9 +18,10 @@ type IPAMConfig struct { EnableIPv4 bool EnableIPv6 bool - EnableSpiderSubnet bool - EnableStatefulSet bool - EnableKubevirtStaticIP bool + EnableSpiderSubnet bool + EnableStatefulSet bool + EnableKubevirtStaticIP bool + EnableReleaseConflictIPsForStateless bool OperationRetries int OperationGapDuration time.Duration diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index 01d628ff78..e2f49eb182 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -25,6 +25,7 @@ import ( type IPAM interface { Allocate(ctx context.Context, addArgs *models.IpamAddArgs) (*models.IpamAddResponse, error) Release(ctx context.Context, delArgs *models.IpamDelArgs) error + ReleaseIPs(ctx context.Context, delArgs *models.IpamBatchDelArgs) error Start(ctx context.Context) error } diff --git a/pkg/ipam/release.go b/pkg/ipam/release.go index 7dd4315e71..887e7b78f6 100644 --- a/pkg/ipam/release.go +++ b/pkg/ipam/release.go @@ -10,6 +10,8 @@ import ( "time" "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -204,3 +206,74 @@ func (i *ipam) release(ctx context.Context, uid string, details []spiderpoolv2be return nil } + +// ReleaseIPs will release the given IP corresponding NIC whole IPs, +// and we will release the SpiderEndpoint recorded IPs first and release the SpiderIPPool recorded IPs later. +func (i *ipam) ReleaseIPs(ctx context.Context, delArgs *models.IpamBatchDelArgs) (err error) { + log := logutils.FromContext(ctx) + + var pod *corev1.Pod + // we need to have the Pod UID for IP release operation + if len(*delArgs.PodUID) == 0 { + pod, err = i.podManager.GetPodByName(ctx, *delArgs.PodNamespace, *delArgs.PodName, constant.IgnoreCache) + if nil != err { + if apierrors.IsNotFound(err) { + log.Sugar().Warnf("Pod '%s/%s' is not found, skip to release IPs due to no Pod UID", *delArgs.PodNamespace, *delArgs.PodName) + return nil + } + return fmt.Errorf("failed to get Pod '%s/%s', error: %v", *delArgs.PodNamespace, *delArgs.PodName, err) + } + // set Pod UID to parameter + *delArgs.PodUID = string(pod.UID) + } + + // check for release conflict IPs + if delArgs.IsReleaseConflictIPs { + if i.config.EnableReleaseConflictIPsForStateless { + if pod == nil { + pod, err = i.podManager.GetPodByName(ctx, *delArgs.PodNamespace, *delArgs.PodName, constant.IgnoreCache) + if nil != err { + return fmt.Errorf("failed to get Pod '%s/%s', error: %v", *delArgs.PodNamespace, *delArgs.PodName, err) + } + } + + podTopController, err := i.podManager.GetPodTopController(ctx, pod) + if nil != err { + return fmt.Errorf("failed to get the top controller of the Pod %s/%s, error: %v", pod.Namespace, pod.Name, err) + } + + // do not release conflict IPs for stateful Pod + if (i.config.EnableStatefulSet && podTopController.APIVersion == appsv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindStatefulSet) || + (i.config.EnableKubevirtStaticIP && podTopController.APIVersion == kubevirtv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindKubevirtVMI) { + log.Warn("no need to release conflict IPs for stateful Pod") + // return error for 'IsReleaseConflictIPs' + return constant.ErrForbidReleasingStatefulWorkload + } + } else { + log.Warn("EnableReleaseConflictIPsForStateless is disabled, skip to release IPs") + // return error for 'IsReleaseConflictIPs' + return constant.ErrForbidReleasingStatelessWorkload + } + } + + // release stateless workload SpiderEndpoint IPs + endpoint, err := i.endpointManager.GetEndpointByName(ctx, *delArgs.PodNamespace, *delArgs.PodName, constant.IgnoreCache) + if nil != err { + return fmt.Errorf("failed to get SpiderEndpoint '%s/%s', error: %v", *delArgs.PodNamespace, *delArgs.PodName, err) + } + recordedIPAllocationDetails, err := i.endpointManager.ReleaseEndpointIPs(ctx, endpoint, *delArgs.PodUID) + if nil != err { + return fmt.Errorf("failed to release SpiderEndpoint IPs, error: %v", err) + } + + // release IPPool IPs + if len(recordedIPAllocationDetails) != 0 { + log.Sugar().Infof("try to release IPs: %v", recordedIPAllocationDetails) + err := i.release(ctx, *delArgs.PodUID, recordedIPAllocationDetails) + if nil != err { + return err + } + } + + return nil +} diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spiderendpoint_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spiderendpoint_types.go index e8da2ab408..b46ff852a7 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spiderendpoint_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spiderendpoint_types.go @@ -46,6 +46,7 @@ type IPAllocationDetail struct { // +kubebuilder:validation:Optional IPv6Pool *string `json:"ipv6Pool,omitempty"` + // DEPRECATED: Vlan is deprecated. // +kubebuilder:default=0 // +kubebuilder:validation:Maximum=4094 // +kubebuilder:validation:Minimum=0 diff --git a/pkg/networking/ipchecking/ipchecking.go b/pkg/networking/ipchecking/ipchecking.go index eb7f91e691..c3d5041e80 100644 --- a/pkg/networking/ipchecking/ipchecking.go +++ b/pkg/networking/ipchecking/ipchecking.go @@ -17,8 +17,10 @@ import ( "github.com/mdlayher/arp" "github.com/mdlayher/ethernet" "github.com/mdlayher/ndp" - "github.com/spidernet-io/spiderpool/pkg/errgroup" "go.uber.org/zap" + + "github.com/spidernet-io/spiderpool/pkg/constant" + "github.com/spidernet-io/spiderpool/pkg/errgroup" ) type IPChecker struct { @@ -160,15 +162,15 @@ func (ipc *IPChecker) ipCheckingByARP() error { ticker := time.NewTicker(ipc.interval) defer ticker.Stop() - stop := false - for i := 0; i < ipc.retries && !stop; i++ { +END: + for i := 0; i < ipc.retries; i++ { select { case <-ctx.Done(): - stop = true + break END case <-ticker.C: err = ipc.arpClient.WriteTo(packet, ethernet.Broadcast) if err != nil { - stop = true + break END } } } @@ -180,8 +182,8 @@ func (ipc *IPChecker) ipCheckingByARP() error { if conflictingMac != "" { // found ip conflicting ipc.logger.Error("Found IPv4 address conflicting", zap.String("Conflicting IP", ipc.ip4.String()), zap.String("Host", conflictingMac)) - return fmt.Errorf("pod's interface %s with an conflicting ip %s, %s is located at %s", ipc.ifi.Name, - ipc.ip4.String(), ipc.ip4.String(), conflictingMac) + return fmt.Errorf("%w: pod's interface %s with an conflicting ip %s, %s is located at %s", + constant.ErrIPConflict, ipc.ifi.Name, ipc.ip4.String(), ipc.ip4.String(), conflictingMac) } ipc.logger.Debug("No ipv4 address conflict", zap.String("IPv4 address", ipc.ip4.String())) @@ -211,8 +213,8 @@ func (ipc *IPChecker) ipCheckingByNDP() error { if err.Error() == NDPFoundReply.Error() { if replyMac != ipc.ifi.HardwareAddr.String() { ipc.logger.Error("Found IPv6 address conflicting", zap.String("Conflicting IP", ipc.ip6.String()), zap.String("Host", replyMac)) - return fmt.Errorf("pod's interface %s with an conflicting ip %s, %s is located at %s", ipc.ifi.Name, - ipc.ip6.String(), ipc.ip6.String(), replyMac) + return fmt.Errorf("%w: pod's interface %s with an conflicting ip %s, %s is located at %s", + constant.ErrIPConflict, ipc.ifi.Name, ipc.ip6.String(), ipc.ip6.String(), replyMac) } } } diff --git a/pkg/workloadendpointmanager/workloadendpoint_manager.go b/pkg/workloadendpointmanager/workloadendpoint_manager.go index 55b731560c..246c9d2c96 100644 --- a/pkg/workloadendpointmanager/workloadendpoint_manager.go +++ b/pkg/workloadendpointmanager/workloadendpoint_manager.go @@ -31,6 +31,7 @@ type WorkloadEndpointManager interface { PatchIPAllocationResults(ctx context.Context, results []*types.AllocationResult, endpoint *spiderpoolv2beta1.SpiderEndpoint, pod *corev1.Pod, podController types.PodTopController, isMultipleNicWithNoName bool) error ReallocateCurrentIPAllocation(ctx context.Context, uid, nodeName, nic string, endpoint *spiderpoolv2beta1.SpiderEndpoint, isMultipleNicWithNoName bool) error UpdateAllocationNICName(ctx context.Context, endpoint *spiderpoolv2beta1.SpiderEndpoint, nic string) (*spiderpoolv2beta1.PodIPAllocation, error) + ReleaseEndpointIPs(ctx context.Context, endpoint *spiderpoolv2beta1.SpiderEndpoint, uid string) ([]spiderpoolv2beta1.IPAllocationDetail, error) } type workloadEndpointManager struct { @@ -224,3 +225,24 @@ func (em *workloadEndpointManager) UpdateAllocationNICName(ctx context.Context, return &endpoint.Status.Current, nil } + +// ReleaseEndpointIPs will release the SpiderEndpoint status recorded IPs. +func (em *workloadEndpointManager) ReleaseEndpointIPs(ctx context.Context, endpoint *spiderpoolv2beta1.SpiderEndpoint, podUID string) ([]spiderpoolv2beta1.IPAllocationDetail, error) { + log := logutils.FromContext(ctx) + + if endpoint.Status.Current.UID != podUID { + return nil, fmt.Errorf("the SpiderEndpoint recorded PodUID '%s' is unmacthed with the given PodUID '%s'", endpoint.Status.Current.UID, podUID) + } + + recordedIPAllocationDetails := endpoint.Status.Current.IPs + if len(recordedIPAllocationDetails) != 0 { + endpoint.Status.Current.IPs = []spiderpoolv2beta1.IPAllocationDetail{} + log.Sugar().Debugf("try to clean up SpiderEndpoint recorded IPs: %s", endpoint) + err := em.client.Update(ctx, endpoint) + if nil != err { + return nil, err + } + } + + return recordedIPAllocationDetails, nil +} diff --git a/pkg/workloadendpointmanager/workloadendpoint_manager_test.go b/pkg/workloadendpointmanager/workloadendpoint_manager_test.go index ac4987b5bc..53de940005 100644 --- a/pkg/workloadendpointmanager/workloadendpoint_manager_test.go +++ b/pkg/workloadendpointmanager/workloadendpoint_manager_test.go @@ -578,5 +578,66 @@ var _ = Describe("WorkloadEndpointManager", Label("workloadendpoint_manager_test Expect(podIPAllocation.IPs[0].NIC).To(Equal(nic)) }) }) + + Describe("ReleaseEndpointIPs", func() { + It("failed to release SpiderEndpoint IPs due to mismatch the PodUID", func() { + endpointT.Status.Current.UID = string(uuid.NewUUID()) + _, err := endpointManager.ReleaseEndpointIPs(ctx, endpointT, string(uuid.NewUUID())) + Expect(err).To(HaveOccurred()) + }) + + It("no SpiderEndpoint recorded IPs", func() { + podUID := string(uuid.NewUUID()) + + endpointT.Status.Current.UID = podUID + ipAllocationDetails, err := endpointManager.ReleaseEndpointIPs(ctx, endpointT, podUID) + Expect(err).NotTo(HaveOccurred()) + Expect(ipAllocationDetails).To(HaveLen(0)) + }) + + It("failed to update SpiderEndpoint", func() { + patches := gomonkey.ApplyMethodReturn(fakeClient, "Update", constant.ErrUnknown) + defer patches.Reset() + + podUID := string(uuid.NewUUID()) + + endpointT.Status.Current.UID = podUID + endpointT.Status.Current.IPs = []spiderpoolv2beta1.IPAllocationDetail{ + { + NIC: "eth0", + IPv4: pointer.String("172.10.2.3/16"), + }, + } + _, err := endpointManager.ReleaseEndpointIPs(ctx, endpointT, podUID) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(constant.ErrUnknown)) + }) + + It("release SpiderEndpoint recorded IPs successfully", func() { + patches := gomonkey.ApplyMethodReturn(fakeClient, "Update", nil) + defer patches.Reset() + + podUID := string(uuid.NewUUID()) + + endpointT.Status.Current.UID = podUID + endpointT.Status.Current.IPs = []spiderpoolv2beta1.IPAllocationDetail{ + { + NIC: "eth0", + IPv4: pointer.String("172.100.1.2/16"), + IPv6: pointer.String("fd00:172:100::201/64"), + }, + { + NIC: "net1", + IPv4: pointer.String("172.200.1.2/16"), + IPv6: pointer.String("fd00:172:200::201/64"), + }, + } + + ipAllocationDetails, err := endpointManager.ReleaseEndpointIPs(ctx, endpointT, podUID) + Expect(err).NotTo(HaveOccurred()) + Expect(endpointT.Status.Current.IPs).To(HaveLen(0)) + Expect(ipAllocationDetails).To(HaveLen(2)) + }) + }) }) }) diff --git a/test/doc/coordinator.md b/test/doc/coordinator.md index 0ed5e8ccc6..1de017a393 100644 --- a/test/doc/coordinator.md +++ b/test/doc/coordinator.md @@ -1,21 +1,23 @@ # E2E Cases for coordinator -| Case ID | Title | Priority | Smoke | Status | Other | -| ------- | ------------------------------------------------------------ | -------- | ----- | ------ | ----- | -| C00001 | coordinator in tuneMode: underlay works well | p1 | smoke | done | | -| C00002 | coordinator in tuneMode: overlay works well | p1 | smoke | done | | -| C00003 | coordinator in tuneMode: underlay with two NIC | p1 | smoke | | | -| C00004 | coordinator in tuneMode: overlay with two NIC | p1 | smoke | | | -| C00005 | In overlay mode: specify the NIC (net1) where the default route is located, use 'ip r get 8.8.8.8' to see if default route nic is the specify NIC | p2 | | done | | -| C00006 | In underlay mode: specify the NIC (net1) where the default route is located, use 'ip r get 8.8.8.8' to see if default route nic is the specify NIC | p2 | | done | | -| C00007 | ip conflict detection (ipv4, ipv6) | p2 | | done | | -| C00008 | override pod mac prefix | p2 | | done | | -| C00009 | gateway connection detection | p2 | | done | | -| C00010 | auto clean up the dirty rules(routing\neighborhood) while pod starting | p2 | | | -| C00011 | In the default scenario (Do not specify the NIC where the default route is located in any way) , use 'ip r get 8.8.8.8' to see if default route NIC is `eth0` | p2 | | done | | -| C00012 | In multi-nic case , use 'ip r get and ' to see if src is from pod's eth0, note: only for ipv4. | p2 | | done | | -| C00013 | Support `spec.externalTrafficPolicy` for service in Local mode, it works well | p2 | | | | -| C00014 | Specify the NIC of the default route, but the NIC does not exist | p3 | | | | -| C00015 | In multi-NIC mode, whether the NIC name is random and pods are created normally | p3 | | | | -| C00016 | The table name can be customized by hostRuleTable | p3 | | | | -| C00017 | TunePodRoutes If false, no routing will be coordinated | p3 | | | | +| Case ID | Title | Priority | Smoke | Status | Other | +|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------|--------|-------| +| C00001 | coordinator in tuneMode: underlay works well | p1 | smoke | done | | +| C00002 | coordinator in tuneMode: overlay works well | p1 | smoke | done | | +| C00003 | coordinator in tuneMode: underlay with two NIC | p1 | smoke | | | +| C00004 | coordinator in tuneMode: overlay with two NIC | p1 | smoke | | | +| C00005 | In overlay mode: specify the NIC (net1) where the default route is located, use 'ip r get 8.8.8.8' to see if default route nic is the specify NIC | p2 | | done | | +| C00006 | In underlay mode: specify the NIC (net1) where the default route is located, use 'ip r get 8.8.8.8' to see if default route nic is the specify NIC | p2 | | done | | +| C00007 | ip conflict detection (ipv4, ipv6) | p2 | | done | | +| C00008 | override pod mac prefix | p2 | | done | | +| C00009 | gateway connection detection | p2 | | done | | +| C00010 | auto clean up the dirty rules(routing\neighborhood) while pod starting | p2 | | | +| C00011 | In the default scenario (Do not specify the NIC where the default route is located in any way) , use 'ip r get 8.8.8.8' to see if default route NIC is `eth0` | p2 | | done | | +| C00012 | In multi-nic case , use 'ip r get and ' to see if src is from pod's eth0, note: only for ipv4. | p2 | | done | | +| C00013 | Support `spec.externalTrafficPolicy` for service in Local mode, it works well | p2 | | | | +| C00014 | Specify the NIC of the default route, but the NIC does not exist | p3 | | | | +| C00015 | In multi-NIC mode, whether the NIC name is random and pods are created normally | p3 | | | | +| C00016 | The table name can be customized by hostRuleTable | p3 | | | | +| C00017 | TunePodRoutes If false, no routing will be coordinated | p3 | | | | +| C00018 | The conflict IPs for stateless Pod should be released | p3 | | done | | +| C00019 | The conflict IPs for stateful Pod should not be released | p3 | | | | diff --git a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go index 60065c45ef..677e148a96 100644 --- a/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go +++ b/test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go @@ -1,5 +1,6 @@ // Copyright 2023 Authors of spidernet-io // SPDX-License-Identifier: Apache-2.0 + package macvlan_overlay_one_test import ( @@ -17,6 +18,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spidernet-io/e2eframework/tools" + errors2 "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -498,7 +500,6 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - VlanID: ptr.To(int32(200)), }, CoordinatorConfig: &spiderpoolv2beta1.CoordinatorSpec{ Mode: &mode, @@ -525,6 +526,28 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") It("It should be possible to detect ip conflicts and log output", Label("C00007", "V00007"), func() { podAnno := types.AnnoPodIPPoolValue{} + var vlanID int32 = 200 + Eventually(func() error { + var smc spiderpoolv2beta1.SpiderMultusConfig + err := frame.KClient.Get(context.TODO(), apitypes.NamespacedName{ + Namespace: namespace, + Name: multusNadName, + }, &smc) + if nil != err { + return err + } + + Expect(smc.Spec.MacvlanConfig).NotTo(BeNil()) + smc.Spec.MacvlanConfig.VlanID = ptr.To(vlanID) + + err = frame.KClient.Update(context.TODO(), &smc) + if nil != err { + return err + } + GinkgoWriter.Printf("update SpiderMultusConfig %s/%s with vlanID %d successfully\n", namespace, multusNadName, vlanID) + return nil + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(BeNil()) + if frame.Info.IpV4Enabled { spiderPoolIPv4SubnetVlan200, err := common.GetIppoolByName(frame, common.SpiderPoolIPv4SubnetVlan200) Expect(err).NotTo(HaveOccurred(), "failed to get v4 ippool, error is %v", err) @@ -668,6 +691,174 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator") return frame.CheckPodListRunning(newPodList) }, common.PodStartTimeout, common.ForcedWaitingTime).Should(BeTrue()) }) + + It("The conflict IPs for stateless Pod should be released", Label("C00018"), func() { + ctx := context.TODO() + + // 1. check the spiderpool-agent ENV SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS enabled or missed + const SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS = "SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS" + spiderpoolAgentDS, err := frame.GetDaemonSet(constant.SpiderpoolAgent, "kube-system") + Expect(err).NotTo(HaveOccurred()) + Expect(spiderpoolAgentDS.Spec.Template.Spec.Containers).To(HaveLen(1)) + + // the release conflicted IPs feature is default to be true if we do not set the ENV + isReleaseConflictIPs := true + for _, env := range spiderpoolAgentDS.Spec.Template.Spec.Containers[0].Env { + if env.Name == SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS { + parseBool, err := strconv.ParseBool(env.Value) + Expect(err).NotTo(HaveOccurred()) + isReleaseConflictIPs = parseBool + break + } + } + + if !isReleaseConflictIPs { + Skip("release conflicted IPs feature is disabled, skip this e2e case") + } + + podAnno := types.AnnoPodIPPoolValue{} + // 2. create an IPPool with conflicted IPs + var conflictV4Pool spiderpoolv2beta1.SpiderIPPool + var firstConflictV4IP string + if frame.Info.IpV4Enabled { + conflictV4PoolName := "conflict-v4-pool" + spiderPoolIPv4PoolDefault, err := common.GetIppoolByName(frame, common.SpiderPoolIPv4PoolDefault) + Expect(err).NotTo(HaveOccurred(), "failed to get ippool %s, error is %v", common.SpiderPoolIPv4PoolDefault, err) + firstDefaultV4IP := strings.Split(spiderPoolIPv4PoolDefault.Spec.IPs[0], "-")[0] + ipv4Prefix, found := strings.CutSuffix(firstDefaultV4IP, ".40.2") + Expect(found).To(BeTrue()) + conflictV4IPs := fmt.Sprintf("%s.41.2-%s.41.4", ipv4Prefix, ipv4Prefix) + GinkgoWriter.Printf("Generate conflict IPv4 IPs: %s\n", conflictV4IPs) + firstConflictV4IP = strings.Split(conflictV4IPs, "-")[0] + + Eventually(func() error { + if !frame.Info.SpiderSubnetEnabled { + return nil + } + + var v4Subnet spiderpoolv2beta1.SpiderSubnet + err := frame.KClient.Get(ctx, apitypes.NamespacedName{Name: common.SpiderPoolIPv4SubnetDefault}, &v4Subnet) + if nil != err { + if errors2.IsNotFound(err) { + return nil + } + return err + } + GinkgoWriter.Printf("try to add IP %s to SpiderSubnet %s\n", firstConflictV4IP, common.SpiderPoolIPv4SubnetDefault) + v4Subnet.Spec.IPs = append(v4Subnet.Spec.IPs, firstConflictV4IP) + err = frame.KClient.Update(ctx, &v4Subnet) + if nil != err { + return err + } + return nil + }).WithTimeout(time.Minute * 2).WithPolling(time.Second).Should(BeNil()) + + conflictV4Pool.Name = conflictV4PoolName + conflictV4Pool.Spec.Subnet = spiderPoolIPv4PoolDefault.Spec.Subnet + conflictV4Pool.Spec.Gateway = spiderPoolIPv4PoolDefault.Spec.Gateway + conflictV4Pool.Spec.IPs = []string{conflictV4IPs} + err = frame.KClient.Create(ctx, &conflictV4Pool) + Expect(err).NotTo(HaveOccurred()) + + // set an IP address for NIC to mock IP conflict + commandV4Str := fmt.Sprintf("ip addr add %s dev eth0", firstConflictV4IP) + output, err := frame.DockerExecCommand(ctx, common.VlanGatewayContainer, commandV4Str) + Expect(err).NotTo(HaveOccurred(), "Failed to exec %s for Node %s, error is: %v, log: %v", commandV4Str, common.VlanGatewayContainer, err, string(output)) + + podAnno.IPv4Pools = []string{conflictV4PoolName} + } + + var conflictV6Pool spiderpoolv2beta1.SpiderIPPool + var firstConflictV6IP string + if frame.Info.IpV6Enabled { + conflictV6PoolName := "conflict-v6-pool" + + spiderPoolIPv6PoolDefault, err := common.GetIppoolByName(frame, common.SpiderPoolIPv6PoolDefault) + Expect(err).NotTo(HaveOccurred(), "failed to get ippool %s, error is %v", common.SpiderPoolIPv6PoolDefault, err) + firstDefaultV6IP := strings.Split(spiderPoolIPv6PoolDefault.Spec.IPs[0], "-")[0] + ipv6Prefix, found := strings.CutSuffix(firstDefaultV6IP, ":f::2") + Expect(found).To(BeTrue()) + conflictV6IPs := fmt.Sprintf("%s:e::2-%s:e::4", ipv6Prefix, ipv6Prefix) + GinkgoWriter.Printf("Generate conflict IPv6 IPs: %s\n", conflictV6IPs) + firstConflictV6IP = strings.Split(conflictV6IPs, "-")[0] + + Eventually(func() error { + if !frame.Info.SpiderSubnetEnabled { + return nil + } + + var v6Subnet spiderpoolv2beta1.SpiderSubnet + err := frame.KClient.Get(ctx, apitypes.NamespacedName{Name: common.SpiderPoolIPv6SubnetDefault}, &v6Subnet) + if nil != err { + if errors2.IsNotFound(err) { + return nil + } + return err + } + GinkgoWriter.Printf("try to add IP %s to SpiderSubnet %s\n", firstConflictV6IP, common.SpiderPoolIPv4SubnetDefault) + v6Subnet.Spec.IPs = append(v6Subnet.Spec.IPs, firstConflictV6IP) + err = frame.KClient.Update(ctx, &v6Subnet) + if nil != err { + return err + } + + return nil + }).WithTimeout(time.Minute * 2).WithPolling(time.Second).Should(BeNil()) + + conflictV6Pool.Name = conflictV6PoolName + conflictV6Pool.Spec.Subnet = spiderPoolIPv6PoolDefault.Spec.Subnet + conflictV6Pool.Spec.Gateway = spiderPoolIPv6PoolDefault.Spec.Gateway + conflictV6Pool.Spec.IPs = []string{conflictV6IPs} + err = frame.KClient.Create(ctx, &conflictV6Pool) + Expect(err).NotTo(HaveOccurred()) + + // set an IP address for NIC to mock IP conflict + commandV6Str := fmt.Sprintf("ip addr add %s dev eth0", firstConflictV6IP) + output, err := frame.DockerExecCommand(ctx, common.VlanGatewayContainer, commandV6Str) + Expect(err).NotTo(HaveOccurred(), "Failed to exec %s for Node %s, error is: %v, log: %v", commandV6Str, common.VlanGatewayContainer, err, string(output)) + + podAnno.IPv6Pools = []string{conflictV6PoolName} + } + + podAnnoMarshal, err := json.Marshal(podAnno) + Expect(err).NotTo(HaveOccurred()) + + // 3. create a pod with conflicted IPs + anno := make(map[string]string) + anno[common.MultusDefaultNetwork] = fmt.Sprintf("%s/%s", namespace, multusNadName) + anno[constant.AnnoPodIPPool] = string(podAnnoMarshal) + deployObject := common.GenerateExampleDeploymentYaml(depName, namespace, int32(1)) + deployObject.Spec.Template.Annotations = anno + ctx, cancel := context.WithTimeout(context.Background(), common.PodStartTimeout) + defer cancel() + GinkgoWriter.Printf("try to create Pod with conflicted IPs IPPool") + _, err = common.CreateDeployUntilExpectedReplicas(frame, deployObject, ctx) + Expect(err).NotTo(HaveOccurred()) + + // 4. delete the Deployments + GinkgoWriter.Printf("The Pod finally runs, task done.") + Expect(frame.DeleteDeployment(depName, namespace)).NotTo(HaveOccurred()) + + // TODO(Icarus9913): create a StatefulSet with conflict IPs and it won't set up successfully. (C00019) + + // 5. delete the conflict IPPools + if frame.Info.IpV4Enabled { + err := frame.KClient.Delete(ctx, &conflictV4Pool) + Expect(err).NotTo(HaveOccurred()) + + commandV4Str := fmt.Sprintf("ip addr del %s dev eth0", firstConflictV4IP) + output, err := frame.DockerExecCommand(ctx, common.VlanGatewayContainer, commandV4Str) + Expect(err).NotTo(HaveOccurred(), "Failed to exec %s for Node %s, error is: %v, log: %v", commandV4Str, common.VlanGatewayContainer, err, string(output)) + } + if frame.Info.IpV6Enabled { + err := frame.KClient.Delete(ctx, &conflictV6Pool) + Expect(err).NotTo(HaveOccurred()) + + commandV6Str := fmt.Sprintf("ip addr del %s dev eth0", firstConflictV6IP) + output, err := frame.DockerExecCommand(ctx, common.VlanGatewayContainer, commandV6Str) + Expect(err).NotTo(HaveOccurred(), "Failed to exec %s for Node %s, error is: %v, log: %v", commandV6Str, common.VlanGatewayContainer, err, string(output)) + } + }) }) Context("Test ip rule and default route are as expected.", func() {