Re-implemented DLB cleanup in nsx-operator

Delete DLB resources.
The size of load balancer service can be, SMALL, MEDIUM, LARGE, XLARGE, or DLB.
The first four sizes are realized on Edge node as a
centralized load balancer. DLB is realized on each ESXi hypervisor as a distributed load balancer.
Previously, this cleanup function was implemented in NCP nsx_policy_cleanup.py.
Now, it is re-implemented in nsx-operator pkg/clean/.

Signed-off-by: Xie Zheng <[email protected]>

cmd_clean/main.go

@@ -63,7 +63,6 @@ func main() {
 	flag.IntVar(&config.LogLevel, "log-level", 0, "Use zap-core log system.")
 	flag.Parse()
 
-	logf.SetLogger(logger.ZapLogger())
 	cf = config.NewNSXOpertorConfig()
 	cf.NsxApiManagers = []string{mgrIp}
 	cf.VCUser = vcUser

pkg/clean/clean.go

@@ -11,7 +11,6 @@ import (
 	"k8s.io/client-go/util/retry"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
-	commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
@@ -28,6 +27,7 @@ var log = logger.Log
 
 // Clean cleans up NSX resources,
 // including security policy, static route, subnet, subnet port, subnet set, vpc, ip pool, nsx service account
+// besides, it also cleans up DLB resources, which was previously implemented in nsx-ncp,
 // it is usually used when nsx-operator is uninstalled and remove all the resources created by nsx-operator
 // return error if any, return nil if no error
 // the error type include followings:
@@ -55,6 +55,22 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig) error {
 			}
 		}
 	}
+	// delete DLB group -> delete virtual servers -> DLB services -> DLB pools -> persistent profiles for DLB
+	if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
+		if err != nil {
+			log.Info("retrying to clean up DLB resources", "error", err)
+			return true
+		}
+		return false
+	}, func() error {
+		if err := CleanDLB(ctx, nsxClient.Cluster, cf); err != nil {
+			return fmt.Errorf("failed to clean up specific resource: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
 	log.Info("cleanup NSX resources successfully")
 	return nil
 }
@@ -91,9 +107,6 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er
 	}
 	vpcService, vpcErr := vpc.InitializeVPC(commonService)
 
-	vpcService, vpcErr := vpc.InitializeVPC(commonService)
-	commonctl.ServiceMediator.VPCService = vpcService
-
 	// initialize all the CR services
 	// Use Fluent Interface to escape error check hell
 

pkg/clean/clean_dlb.go

@@ -0,0 +1,94 @@
+package clean
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	neturl "net/url"
+	"strings"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
+)
+
+type (
+	mapInterface = map[string]interface{}
+)
+
+const TagDLB = "DLB"
+
+func appendIfNotExist(slice []string, s string) []string {
+	for _, item := range slice {
+		if item == s {
+			return slice
+		}
+	}
+	return append(slice, s)
+}
+
+func httpQueryDLBResources(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) {
+	queryParam := "resource_type:" + resource +
+		"&tags.scope:ncp\\/cluster" +
+		"&tags.tag:" + cf.Cluster +
+		"&tags.scope:ncp\\/created_for" +
+		"&tags.tag:" + TagDLB
+
+	pairs := strings.Split(queryParam, "&")
+	params := make(map[string]string)
+	for _, pair := range pairs {
+		kv := strings.Split(pair, ":")
+		if len(kv) == 2 {
+			params[kv[0]] = kv[1]
+		}
+	}
+	var encodedPairs []string
+	for key, value := range params {
+		encodedKey := neturl.QueryEscape(key)
+		encodedValue := neturl.QueryEscape(value)
+		encodedPairs = append(encodedPairs, fmt.Sprintf("%s:%s", encodedKey, encodedValue))
+	}
+
+	encodedQuery := strings.Join(encodedPairs, "%20AND%20")
+	url := "api/v1/search/query?query=" + encodedQuery
+
+	resp, err := cluster.HttpGet(url)
+	if err != nil {
+		return nil, err
+	}
+	var resourcePath []string
+	for _, item := range resp["results"].([]interface{}) {
+		resourcePath = appendIfNotExist(resourcePath, item.(mapInterface)["path"].(string))
+	}
+	return resourcePath, nil
+}
+
+func CleanDLB(ctx context.Context, cluster *nsx.Cluster, cf *config.NSXOperatorConfig) error {
+	log.Info("Deleting DLB resources started")
+
+	resources := []string{"Group", "LBVirtualServer", "LBService", "LBPool", "LBCookiePersistenceProfile"}
+	var allPaths []string
+
+	for _, resource := range resources {
+		paths, err := httpQueryDLBResources(cluster, cf, resource)
+		if err != nil {
+			return err
+		}
+		log.Info(resource, "count", len(paths))
+		allPaths = append(allPaths, paths...)
+	}
+
+	log.Info("Deleting DLB resources", "paths", allPaths)
+	for _, path := range allPaths {
+		url := "api/v1" + path
+		select {
+		case <-ctx.Done():
+			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
+		default:
+			if err := cluster.HttpDelete(url); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/clean/types.go

@@ -31,29 +31,3 @@ func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService {
 	c.cleans = append(c.cleans, clean)
 	return c
 }
-
-type cleanupFunc func() (cleanup, error)
-
-type CleanupService struct {
-	cleans []cleanup
-	err    error
-}
-
-func NewCleanupService() *CleanupService {
-	return &CleanupService{}
-}
-
-func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService {
-	var clean cleanup
-	if c.err != nil {
-		return c
-	}
-
-	clean, c.err = f()
-	if c.err != nil {
-		return c
-	}
-
-	c.cleans = append(c.cleans, clean)
-	return c
-}

pkg/config/config.go

@@ -281,12 +281,6 @@ func (vcConfig *VCConfig) validate() error {
 		configLog.Info("validate VcConfig failed VCUser %s VCPassword %s", vcConfig.VCUser, vcConfig.VCPassword)
 		return err
 	}
-	// VCPassword, VCUser should be both empty or valid
-	if !((len(vcConfig.VCPassword) > 0) == (len(vcConfig.VCUser) > 0)) {
-		err := errors.New("invalid field " + "VCUser, VCPassword")
-		log.Info("validate VcConfig failed", "VCUser", vcConfig.VCUser, "VCPassword", vcConfig.VCPassword)
-		return err
-	}
 	return nil
 }