change force delete wait time mins & increase time to fire NS alert log

Azure · Jan 21, 2025 · fed44c4 · fed44c4
1 parent 9f4a632
commit fed44c4
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 11 deletions.
diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml
@@ -16,7 +16,7 @@ enableWebhook: true
 webhookServiceName: fleetwebhook
 enableGuardRail: true
 webhookClientConnectionType: service
-forceDeleteWaitTime: 15m0s
+forceDeleteWaitTime: 5m0s
 clusterUnhealthyThreshold: 3m0s
 namespace:
   fleet-system

diff --git a/cmd/hubagent/options/options.go b/cmd/hubagent/options/options.go
@@ -144,7 +144,7 @@ func (o *Options) AddFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&o.EnableV1Alpha1APIs, "enable-v1alpha1-apis", false, "If set, the agents will watch for the v1alpha1 APIs.")
 	flags.BoolVar(&o.EnableV1Beta1APIs, "enable-v1beta1-apis", true, "If set, the agents will watch for the v1beta1 APIs.")
 	flags.BoolVar(&o.EnableClusterInventoryAPIs, "enable-cluster-inventory-apis", false, "If set, the agents will watch for the ClusterInventory APIs.")
-	flags.DurationVar(&o.ForceDeleteWaitTime.Duration, "force-delete-wait-time", 15*time.Minute, "The duration the hub agent waits before force deleting a member cluster.")
+	flags.DurationVar(&o.ForceDeleteWaitTime.Duration, "force-delete-wait-time", 5*time.Minute, "The duration the hub agent waits before force deleting a member cluster.")
 	flags.BoolVar(&o.EnableStagedUpdateRunAPIs, "enable-staged-update-run-apis", false, "If set, the agents will watch for the ClusterStagedUpdateRun APIs.")
 	flags.BoolVar(&o.EnableEvictionAPIs, "enable-eviction-apis", false, "If set, the agents will watch for the Eviction and PlacementDisruptionBudget APIs.")
 

diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller.go
@@ -141,8 +141,8 @@ func (r *Reconciler) handleDelete(ctx context.Context, mc *clusterv1beta1.Member
 	if !currentNS.DeletionTimestamp.IsZero() {
 		klog.V(2).InfoS("The member cluster namespace is still being deleted", "memberCluster", mcObjRef, "deleteTimestamp", currentNS.DeletionTimestamp)
 		var stuckErr error
-		if time.Now().After(currentNS.DeletionTimestamp.Add(5 * time.Minute)) {
-			// alert if the namespace is stuck in deleting for more than 5 minutes
+		if time.Now().After(currentNS.DeletionTimestamp.Add(10 * time.Minute)) {
+			// alert if the namespace is stuck in deleting for more than 10 minutes
 			stuckErr = controller.NewUnexpectedBehaviorError(fmt.Errorf("the member cluster namespace %s has been deleting since %s", namespaceName, currentNS.DeletionTimestamp.Format(time.RFC3339)))
 		}
 		return runtime.Result{RequeueAfter: time.Second}, stuckErr

diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go
@@ -59,7 +59,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			By("create the member cluster reconciler")
 			r = &Reconciler{
 				Client:              k8sClient,
-				ForceDeleteWaitTime: 15 * time.Minute,
+				ForceDeleteWaitTime: 5 * time.Minute,
 			}
 			err := r.SetupWithManager(mgr)
 			Expect(err).Should(Succeed())
@@ -149,7 +149,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			result, err := r.Reconcile(ctx, ctrl.Request{
 				NamespacedName: memberClusterNamespacedName,
 			})
-			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute}))
+			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime}))
 			Expect(err).Should(Succeed())
 
 			var imc clusterv1beta1.InternalMemberCluster
@@ -206,7 +206,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 
 			By("trigger reconcile again to initiate leave workflow")
 			result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: memberClusterNamespacedName})
-			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute}))
+			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime}))
 			Expect(err).Should(Succeed())
 
 			var imc clusterv1beta1.InternalMemberCluster
@@ -274,7 +274,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			r = &Reconciler{
 				Client:                  k8sClient,
 				NetworkingAgentsEnabled: true,
-				ForceDeleteWaitTime:     15 * time.Minute,
+				ForceDeleteWaitTime:     5 * time.Minute,
 			}
 			err := r.SetupWithManager(mgr)
 			Expect(err).Should(Succeed())
@@ -449,7 +449,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			result, err := r.Reconcile(ctx, ctrl.Request{
 				NamespacedName: memberClusterNamespacedName,
 			})
-			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute}))
+			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime}))
 			Expect(err).Should(Succeed())
 
 			By("getting imc status")
@@ -466,7 +466,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			result, err = r.Reconcile(ctx, ctrl.Request{
 				NamespacedName: memberClusterNamespacedName,
 			})
-			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute}))
+			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime}))
 			Expect(err).Should(Succeed())
 
 			By("checking mc status")
@@ -504,7 +504,7 @@ var _ = Describe("Test MemberCluster Controller", func() {
 			result, err = r.Reconcile(ctx, ctrl.Request{
 				NamespacedName: memberClusterNamespacedName,
 			})
-			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute}))
+			Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime}))
 			Expect(err).Should(Succeed())
 
 			By("checking mc status")