Add AVI allow rule when VPC created

Add AVI allow rule when VPC created so VIP of AVI lb could be accessed
outside the VPC
It will check nsxt version to enable it

Test Done:
Create lb service in one VPC, create pod in other VPC
1. Run the nsx-operator, check if the rule has been added
2. Visit lb service ip from other VPC pod
3. Restart the nsx-operator, check if the rule skip to update since CIDR
    is the same

pkg/clean/clean.go

@@ -58,7 +58,6 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er
 		NSXClient: nsxClient,
 		NSXConfig: cf,
 	}
-
 	vpcService, vpcErr := vpc.InitializeVPC(commonService)
 	commonctl.ServiceMediator.VPCService = vpcService
 

pkg/controllers/vpc/vpc_controller.go

@@ -70,6 +70,12 @@ func (r *VPCReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 			updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client)
 			return common.ResultRequeueAfter10sec, err
 		}
+		err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
+		if err != nil {
+			log.Error(err, "operate failed, would retry exponentially", "VPC", req.NamespacedName)
+			updateFail(r.Service.NSXConfig, &ctx, obj, &err, r.Client)
+			return common.ResultRequeueAfter10sec, err
+		}
 
 		snatIP, path, cidr := "", "", ""
 		// currently, auto snat is not exposed, and use default value True

pkg/nsx/client.go

@@ -23,6 +23,7 @@ import (
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/infra/realized_state"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs"
 	nat "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/nat"
+	vpc_sp "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/security_policies"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ip_pools"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects/vpcs/subnets/ports"
@@ -39,10 +40,11 @@ const (
 	ServiceAccountRestore
 	ServiceAccountCertRotation
 	StaticRoute
+	VpcAviRule
 	AllFeatures
 )
 
-var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE"}
+var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_ACCOUNT", "NSX_SERVICE_ACCOUNT_RESTORE", "NSX_SERVICE_ACCOUNT_CERT_ROTATION", "STATIC_ROUTE", "VPC_AVI_RULE"}
 
 type Client struct {
 	NsxConfig     *config.NSXOperatorConfig
@@ -63,6 +65,10 @@ type Client struct {
 	PrincipalIdentitiesClient  trust_management.PrincipalIdentitiesClient
 	WithCertificateClient      principal_identities.WithCertificateClient
 
+	// for AVI security policy rule
+	VPCSecurityClient vpcs.SecurityPoliciesClient
+	VPCRuleClient     vpc_sp.RulesClient
+
 	OrgRootClient       nsx_policy.OrgRootClient
 	ProjectInfraClient  projects.InfraClient
 	VPCClient           projects.VpcsClient
@@ -94,12 +100,8 @@ type NSXHealthChecker struct {
 }
 
 type NSXVersionChecker struct {
-	cluster                           *Cluster
-	securityPolicySupported           bool
-	nsxServiceAccountSupported        bool
-	nsxServiceAccountRestoreSupported bool
-	vpcSupported                      bool
-	featureSupported                  [AllFeatures]bool
+	cluster          *Cluster
+	featureSupported [AllFeatures]bool
 }
 
 func (ck *NSXHealthChecker) CheckNSXHealth(req *http.Request) error {
@@ -158,6 +160,9 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
 
+	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
+	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
+
 	nsxChecker := &NSXHealthChecker{
 		cluster: cluster,
 	}
@@ -193,6 +198,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		PortClient:         portClient,
 		PortStateClient:    portStateClient,
 		SubnetStatusClient: subnetStatusClient,
+		VPCSecurityClient:  vpcSecurityClient,
+		VPCRuleClient:      vpcRuleClient,
 
 		NSXChecker:          *nsxChecker,
 		NSXVerChecker:       *nsxVersionChecker,
@@ -215,6 +222,10 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		err := errors.New("NSXServiceAccountRestore feature support check failed")
 		log.Error(err, "initial NSX version check for NSXServiceAccountRestore got error")
 	}
+	if !nsxClient.NSXCheckVersion(VpcAviRule) {
+		err := errors.New("VpcAviRule feature support check failed")
+		log.Error(err, "initial NSX version check for VpcAviRule got error")
+	}
 	if !nsxClient.NSXCheckVersion(ServiceAccountCertRotation) {
 		err := errors.New("ServiceAccountCertRotation feature support check failed")
 		log.Error(err, "initial NSX version check for ServiceAccountCertRotation got error")
@@ -247,3 +258,7 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 	client.NSXVerChecker.featureSupported[feature] = true
 	return true
 }
+
+func (client *Client) FeatureEnabled(feature int) bool {
+	return client.NSXVerChecker.featureSupported[feature] == true
+}

pkg/nsx/cluster.go

@@ -331,6 +331,9 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	case ServiceAccountCertRotation:
 		minVersion = nsx413Version
 		validFeature = true
+	case VpcAviRule:
+		minVersion = nsx411Version
+		validFeature = true
 	}
 
 	if validFeature {

pkg/nsx/services/common/store.go

@@ -132,40 +132,15 @@ func (service *Service) InitializeVPCResourceStore(wg *sync.WaitGroup, fatalErro
 	service.InitializeCommonStore(wg, fatalErrors, org, project, resourceTypeValue, tags, store)
 }
 
-// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore
-func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) {
-	defer wg.Done()
-
-	tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1)
-	tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1)
-	tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue)
-
-	for _, tag := range tags {
-		tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1)
-		tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey)
-		if tag.Tag != nil {
-			tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1)
-			tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue)
-		}
-	}
-
-	resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue)
-	queryParam := resourceParam + " AND " + tagParam
-
-	if org != "" || project != "" {
-		// QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A",
-		//"path%25%3A" would fail to get response. Hack it here.
-		path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*"
-		pathUnescape, _ := url.PathUnescape("path%3A")
-		queryParam += " AND " + pathUnescape + path
-	}
-	queryParam += " AND marked_for_delete:false"
+type Filter func(interface{}) *data.StructValue
 
+// PopulateResourcetoStore is the method used by populating resources created not by nsx-operator
+func (service *Service) PopulateResourcetoStore(wg *sync.WaitGroup, fatalErrors chan error, resourceTypeValue string, queryParam string, store Store, filter Filter) {
+	defer wg.Done()
 	var cursor *string = nil
 	count := uint64(0)
 	for {
 		var err error
-
 		var results []*data.StructValue
 		var resultCount *int64
 		if store.IsPolicyAPI() {
@@ -181,6 +156,7 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch
 			resultCount = response.ResultCount
 			err = searchEerr
 		}
+
 		err = TransError(err)
 		if _, ok := err.(nsxutil.PageMaxError); ok == true {
 			DecrementPageSize(Int64(PageSize))
@@ -190,6 +166,9 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch
 			fatalErrors <- err
 		}
 		for _, entity := range results {
+			if filter != nil {
+				entity = filter(entity)
+			}
 			err = store.TransResourceToStore(entity)
 			if err != nil {
 				fatalErrors <- err
@@ -206,3 +185,32 @@ func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors ch
 	}
 	log.Info("initialized store", "resourceType", resourceTypeValue, "count", count)
 }
+
+// InitializeCommonStore is the common method used by InitializeResourceStore and InitializeVPCResourceStore
+func (service *Service) InitializeCommonStore(wg *sync.WaitGroup, fatalErrors chan error, org string, project string, resourceTypeValue string, tags []model.Tag, store Store) {
+	tagScopeClusterKey := strings.Replace(TagScopeCluster, "/", "\\/", -1)
+	tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1)
+	tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue)
+
+	for _, tag := range tags {
+		tagKey := strings.Replace(*tag.Scope, "/", "\\/", -1)
+		tagParam += fmt.Sprintf(" AND tags.scope:%s ", tagKey)
+		if tag.Tag != nil {
+			tagValue := strings.Replace(*tag.Tag, ":", "\\:", -1)
+			tagParam += fmt.Sprintf(" AND tags.tag:%s ", tagValue)
+		}
+	}
+
+	resourceParam := fmt.Sprintf("%s:%s", ResourceType, resourceTypeValue)
+	queryParam := resourceParam + " AND " + tagParam
+
+	if org != "" || project != "" {
+		// QueryClient.List() will escape the path, "path:" then will be "path%25%3A" instead of "path:3A",
+		//"path%25%3A" would fail to get response. Hack it here.
+		path := "\\/orgs\\/" + org + "\\/projects\\/" + project + "\\/*"
+		pathUnescape, _ := url.PathUnescape("path%3A")
+		queryParam += " AND " + pathUnescape + path
+	}
+	queryParam += " AND marked_for_delete:false"
+	service.PopulateResourcetoStore(wg, fatalErrors, resourceTypeValue, queryParam, store, nil)
+}

pkg/nsx/services/common/types.go

@@ -62,6 +62,7 @@ const (
 	TagValueGroupScope              string = "scope"
 	TagValueGroupSrc                string = "source"
 	TagValueGroupDst                string = "destination"
+	TagValueGroupAvi                string = "avi"
 	AnnotationVPCNetworkConfig      string = "nsx.vmware.com/vpc_network_config"
 	AnnotationVPCName               string = "nsx.vmware.com/vpc_name"
 	AnnotationPodMAC                string = "nsx.vmware.com/mac"

pkg/nsx/services/vpc/store.go

@@ -147,3 +147,86 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB
 	block := indexResults[0].((model.IpAddressBlock))
 	return &block
 }
+
+// keyFuncAVI is used to get the key of a AVI rule related resource
+func keyFuncAVI(obj interface{}) (string, error) {
+	switch v := obj.(type) {
+	case model.Rule:
+		return *v.Path, nil
+	case model.SecurityPolicy:
+		return *v.Path, nil
+	case model.Group:
+		return *v.Path, nil
+	case model.IpAddressBlock:
+		return *v.Path, nil
+	default:
+		return "", errors.New("keyFunc doesn't support unknown type")
+	}
+}
+
+// AviRuleStore is a store for saving AVI related Rules in VPCs
+type AviRuleStore struct {
+	common.ResourceStore
+}
+
+func (ruleStore *AviRuleStore) Apply(i interface{}) error {
+	return nil
+}
+func (ruleStore *AviRuleStore) GetByKey(key string) *model.Rule {
+	obj := ruleStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		rule := obj.(model.Rule)
+		return &rule
+	}
+	return nil
+}
+
+// PubIPblockStore is a store to query external ip blocks cidr
+type PubIPblockStore struct {
+	common.ResourceStore
+}
+
+func (ipBlockStore *PubIPblockStore) Apply(i interface{}) error {
+	return nil
+}
+func (ipBlockStore *PubIPblockStore) GetByKey(key string) *model.IpAddressBlock {
+	obj := ipBlockStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		ipblock := obj.(model.IpAddressBlock)
+		return &ipblock
+	}
+	return nil
+}
+
+type AviGroupStore struct {
+	common.ResourceStore
+}
+
+func (groupStore *AviGroupStore) Apply(i interface{}) error {
+	return nil
+}
+func (groupStore *AviGroupStore) GetByKey(key string) *model.Group {
+	obj := groupStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		group := obj.(model.Group)
+		return &group
+	}
+	return nil
+}
+
+type AviSecurityPolicyStore struct {
+	common.ResourceStore
+}
+
+func (securityPolicyStore *AviSecurityPolicyStore) Apply(i interface{}) error {
+	return nil
+}
+
+func (securityPolicyStore *AviSecurityPolicyStore) GetByKey(key string) *model.SecurityPolicy {
+	obj := securityPolicyStore.ResourceStore.GetByKey(key)
+	if obj != nil {
+		sp := obj.(model.SecurityPolicy)
+		return &sp
+	}
+	return nil
+}