check license from nsxt side

While init, it will check the license from nsxt side. If CONTAINER
license is disable, it will reboot. If DFW license is disable,
security policy will be only response for DELETE operation.
It will check license periodically.

Test Done:
no CONTAINER license
1. if no CONTAINER license, nsx-operator should reset
CONTAINER license enable, DFW disable
1. nsx-operator could bootup
2. security policy failed to create or update
3. security policy could be deleted
CONTAINER license enable, DFW enable -> CONTAINER/DFW disable
1. nsx-operator restart due to DFW changed

cmd/main.go

@@ -169,6 +169,11 @@ func main() {
 		NSXClient: nsxClient,
 		NSXConfig: cf,
 	}
+	err = commonService.NSXClient.ValidateLicense(true)
+	if err != nil {
+		os.Exit(1)
+	}
+	go updateLicensePeriodically(nsxClient, metrics.LicenseTimeout*time.Second)
 
 	var vpcService *vpc.VPCService
 
@@ -263,3 +268,15 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) {
 		}
 	}
 }
+
+func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) {
+	for {
+		select {
+		case <-time.After(interval):
+		}
+		err := nsxClient.ValidateLicense(false)
+		if err != nil {
+			os.Exit(1)
+		}
+	}
+}

pkg/metrics/metrics.go

@@ -22,6 +22,8 @@ const (
 	ControllerDeleteSuccessTotalKey = "controller_delete_success_total"
 	ControllerDeleteFailTotalKey    = "controller_delete_fail_total"
 	ScrapeTimeout                   = 30
+	// LicenseTimeout is the timeout for checking license status.
+	LicenseTimeout = 2 * 3600
 )
 
 var log = logger.Log

pkg/nsx/client.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 const (
@@ -264,3 +265,31 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 func (client *Client) FeatureEnabled(feature int) bool {
 	return client.NSXVerChecker.featureSupported[feature] == true
 }
+
+// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not
+// if not init, nsx-operator will check if license has been updated.
+// once license updated, operator will restart
+// if FeatureContainer license is false, operatore will restart
+func (client *Client) ValidateLicense(init bool) error {
+	log.Info("Checking NSX license")
+	oldContainerLicense := util.IsLicensed(util.FeatureContainer)
+	oldDfwLicense := util.IsLicensed(util.FeatureDFW)
+	err := client.NSXChecker.cluster.FetchLicense()
+	if err != nil {
+		return err
+	}
+	if !util.IsLicensed(util.FeatureContainer) {
+		err = errors.New("NSX license check failed")
+		log.Error(err, "container license is not supported")
+		return err
+	}
+	if !init {
+		newContainerLicense := util.IsLicensed(util.FeatureContainer)
+		newDfwLicense := util.IsLicensed(util.FeatureDFW)
+		if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense {
+			log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense)
+			return errors.New("license updated")
+		}
+	}
+	return nil
+}

pkg/nsx/cluster.go

@@ -5,9 +5,11 @@ package nsx
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"math/big"
 	"net"
 	"net/http"
 	"os"
@@ -39,6 +41,12 @@ const (
 const (
 	EnvoyUrlWithCert       = "http://%s:%d/external-cert/http1/%s"
 	EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s"
+	LicenseAPI             = "api/v1/licenses/licensed-features"
+)
+
+const (
+	maxNSXGetRetries = 10
+	NSXGetDelay      = 2 * time.Second
 )
 
 // Cluster consists of endpoint and provides http.Client used to send http requests.
@@ -418,3 +426,58 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	}
 	return false
 }
+
+func (cluster *Cluster) createHttpRequest(api string, ep *Endpoint) (*http.Request, error) {
+	return http.NewRequest("GET", fmt.Sprintf("%s://%s/%s", ep.Scheme(), ep.Host(), api), nil)
+}
+
+func (cluster *Cluster) getLicenseFromNsx() (*util.NsxLicense, error) {
+	ep := cluster.endpoints[0]
+	req, err := cluster.createHttpRequest(LicenseAPI, ep)
+	if err != nil {
+		log.Error(err, "failed to create http request")
+		return nil, err
+	}
+	err = ep.UpdateHttpRequestAuth(req)
+	if err != nil {
+		log.Error(err, "keep alive update auth error")
+		return nil, err
+	}
+
+	resp, err := ep.noBalancerClient.Do(req)
+	if err != nil {
+		log.Error(err, "failed to get nsx license")
+		return nil, err
+	}
+	nsxLicense := &util.NsxLicense{}
+	err, _ = util.HandleHTTPResponse(resp, nsxLicense, true)
+	return nsxLicense, err
+}
+
+func (cluster *Cluster) getLicenseWithRetries(delay time.Duration, maxRetry int) (*util.NsxLicense, error) {
+	var err error
+	for i := 0; i < maxRetry; i++ {
+		nsxLicense, err := cluster.getLicenseFromNsx()
+		if err != nil {
+			log.Error(err, "failed to get nsx license")
+			rand, err := rand.Int(rand.Reader, big.NewInt(1000))
+			if err != nil {
+				log.Error(err, "failed to generate random number")
+				return nil, err
+			}
+			time.Sleep(delay + time.Duration(rand.Int64())*time.Millisecond)
+		} else {
+			return nsxLicense, nil
+		}
+	}
+	return nil, err
+}
+
+func (cluster *Cluster) FetchLicense() error {
+	nsxLicense, err := cluster.getLicenseWithRetries(NSXGetDelay, maxNSXGetRetries)
+	if err != nil {
+		return err
+	}
+	util.UpdateFeatureLicense(nsxLicense)
+	return nil
+}

pkg/nsx/cluster_test.go

@@ -4,8 +4,11 @@
 package nsx
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -17,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 func TestNewCluster(t *testing.T) {
@@ -344,3 +348,70 @@ func TestCluster_CreateServerUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestGetLicenseFromNsx(t *testing.T) {
+	address := address{
+		host:   "1.2.3.4",
+		scheme: "https",
+	}
+	// Success case
+	cluster := &Cluster{endpoints: []*Endpoint{{
+		provider: &address,
+	}}}
+
+	// Request creation failure
+	patch := gomonkey.ApplyFunc(http.NewRequest,
+		func(method, url string, body io.Reader) (*http.Request, error) {
+			return nil, errors.New("request error")
+		})
+	license, err := cluster.getLicenseFromNsx()
+	assert.Error(t, err)
+	assert.Nil(t, license)
+	patch.Reset()
+
+	// HTTP error
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			return nil, errors.New("http error")
+		})
+
+	license, err = cluster.getLicenseFromNsx()
+	assert.Error(t, err)
+	assert.Nil(t, license)
+	patch.Reset()
+
+	// normal case
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			res := &nsxutil.NsxLicense{
+				Results: []struct {
+					FeatureName string `json:"feature_name"`
+					IsLicensed  bool   `json:"is_licensed"`
+				}{{
+					FeatureName: "CONTAINER",
+					IsLicensed:  true,
+				},
+					{
+						FeatureName: "DFW",
+						IsLicensed:  true,
+					},
+				},
+				ResultCount: 2,
+			}
+
+			jsonBytes, _ := json.Marshal(res)
+
+			return &http.Response{
+				StatusCode: 200,
+				Body:       io.NopCloser(bytes.NewReader(jsonBytes)),
+				Header: http.Header{
+					"Content-Type": []string{"application/json"},
+				},
+				Request: req,
+			}, nil
+		})
+	defer patch.Reset()
+	_, err = cluster.getLicenseFromNsx()
+	assert.Nil(t, err)
+
+}

pkg/nsx/services/securitypolicy/firewall.go

@@ -16,6 +16,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 	"github.com/vmware-tanzu/nsx-operator/pkg/util"
 )
 
@@ -127,6 +128,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi
 }
 
 func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error {
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("no DFW license, skip creating SecurityPolicy.")
+		return nsxutil.RestrictionError{Desc: "no DFW license"}
+	}
 	var err error
 	switch obj.(type) {
 	case *networkingv1.NetworkPolicy:

pkg/nsx/services/vpc/vpc.go

@@ -22,6 +22,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 	"github.com/vmware-tanzu/nsx-operator/pkg/util"
 )
 
@@ -614,6 +615,9 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin
 	if !enableAviAllowRule {
 		return nil
 	}
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		return nil
+	}
 	vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path)
 	if err != nil {
 		log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path)

pkg/nsx/services/vpc/vpc_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) {
 	sp := model.SecurityPolicy{
 		Path: &sppath1,
 	}
+	util.UpdateLicense(util.FeatureDFW, true)
 
 	// security policy not found
 	spClient.SP = sp

pkg/nsx/util/license.go

@@ -0,0 +1,65 @@
+package util
+
+import (
+	"sync"
+)
+
+const (
+	FeatureContainer         = "CONTAINER"
+	FeatureDFW               = "DFW"
+	LicenseContainerNetwork  = "CONTAINER_NETWORK"
+	LicenseContainerSecurity = "DFW"
+	LicenseContainer         = "CONTAINER"
+)
+
+var (
+	container_network   = false
+	container_security  = false
+	licenseMutex        sync.Mutex
+	licenseMap          = map[string]bool{FeatureContainer: container_network, FeatureDFW: container_security}
+	Feature_license_map = map[string][]string{FeatureContainer: {LicenseContainerNetwork,
+		LicenseContainerSecurity},
+		FeatureDFW: {LicenseContainerSecurity}}
+	Features_to_check = []string{FeatureContainer, FeatureDFW}
+)
+
+type NsxLicense struct {
+	Results []struct {
+		FeatureName string `json:"feature_name"`
+		IsLicensed  bool   `json:"is_licensed"`
+	} `json:"results"`
+	ResultCount int `json:"result_count"`
+}
+
+func IsLicensed(feature string) bool {
+	licenseMutex.Lock()
+	defer licenseMutex.Unlock()
+	return licenseMap[feature]
+}
+
+func UpdateLicense(feature string, isLicensed bool) {
+	licenseMutex.Lock()
+	licenseMap[feature] = isLicensed
+	licenseMutex.Unlock()
+}
+
+func searchLicense(licenses *NsxLicense, licenseNames []string) bool {
+	license := false
+	for _, feature := range licenses.Results {
+		for _, licenseName := range licenseNames {
+			if feature.FeatureName == licenseName {
+				license = license || feature.IsLicensed
+			}
+		}
+	}
+	return license
+}
+
+func UpdateFeatureLicense(licenses *NsxLicense) {
+	for _, feature := range Features_to_check {
+		licenseNames := Feature_license_map[feature]
+		license := searchLicense(licenses, licenseNames)
+		UpdateLicense(feature, license)
+		log.V(1).Info("update license", "feature", feature, "license", license)
+	}
+}