Fix bug which should delete node from store when there is node not fo…

…und error

When node is deleted, the phase node.ObjectMeta.DeletionTimestamp being not zero is intermittent,
and it is more possible the node is not found, we should delete the node from store.

pkg/controllers/node/node_controller.go

@@ -45,10 +45,15 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		if errors.IsNotFound(err) {
 			log.Info("node not found", "req", req.NamespacedName)
 			deleted = true
+			if err := r.Service.SyncNodeStore(req.NamespacedName.Name, deleted); err != nil {
+				log.Error(err, "failed to sync node store", "req", req.NamespacedName)
+				return ctrl.Result{}, err
+			}
+			return ctrl.Result{}, nil
 		} else {
 			log.Error(err, "unable to fetch node", "req", req.NamespacedName)
+			return common.ResultNormal, client.IgnoreNotFound(err)
 		}
-		return common.ResultNormal, client.IgnoreNotFound(err)
 	}
 	if common.NodeIsMaster(node) {
 		// For WCP supervisor cluster, the master node isn't a transport node.

pkg/controllers/node/node_controller_test.go

@@ -17,15 +17,20 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/node"
 )
 
 func createMockNodeService() *node.NodeService {
 	return &node.NodeService{
 		Service: servicecommon.Service{
-			NSXConfig: &config.NSXOperatorConfig{
-				NsxConfig: &config.NsxConfig{},
+			NSXClient: &nsx.Client{
+				NsxConfig: &config.NSXOperatorConfig{
+					CoeConfig: &config.CoeConfig{
+						Cluster: "k8scl-one:test",
+					},
+				},
 			},
 		},
 	}
@@ -90,18 +95,28 @@ func TestNodeReconciler_Reconcile(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			ctx := context.Background()
+			nodeName := "test-node"
 
 			if tt.existingNode != nil {
+				nodeName = tt.existingNode.Name
 				err := reconciler.Client.Create(ctx, tt.existingNode)
 				assert.NoError(t, err)
 			}
 
 			req := ctrl.Request{
 				NamespacedName: types.NamespacedName{
-					Name: "test-node",
+					Name: nodeName,
 				},
 			}
 
+			if tt.name != "Master Node" {
+				patchesSyncNodeStore := gomonkey.ApplyFunc((*node.NodeService).SyncNodeStore,
+					func(n *node.NodeService, nodeName string, deleted bool) error {
+						return nil
+					})
+				defer patchesSyncNodeStore.Reset()
+			}
+
 			result, err := reconciler.Reconcile(ctx, req)
 
 			if tt.expectedErr {