From fed44c471fa4a9686daf14be5ee5c18a81b7caac Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 21 Jan 2025 14:01:34 +0530 Subject: [PATCH] change force delete wait time mins & increase time to fire NS alert log --- charts/hub-agent/values.yaml | 2 +- cmd/hubagent/options/options.go | 2 +- .../v1beta1/membercluster_controller.go | 4 ++-- .../membercluster_controller_integration_test.go | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml index 845fa1c93..cef6f5907 100644 --- a/charts/hub-agent/values.yaml +++ b/charts/hub-agent/values.yaml @@ -16,7 +16,7 @@ enableWebhook: true webhookServiceName: fleetwebhook enableGuardRail: true webhookClientConnectionType: service -forceDeleteWaitTime: 15m0s +forceDeleteWaitTime: 5m0s clusterUnhealthyThreshold: 3m0s namespace: fleet-system diff --git a/cmd/hubagent/options/options.go b/cmd/hubagent/options/options.go index d54f8d0cf..1d461a6c2 100644 --- a/cmd/hubagent/options/options.go +++ b/cmd/hubagent/options/options.go @@ -144,7 +144,7 @@ func (o *Options) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&o.EnableV1Alpha1APIs, "enable-v1alpha1-apis", false, "If set, the agents will watch for the v1alpha1 APIs.") flags.BoolVar(&o.EnableV1Beta1APIs, "enable-v1beta1-apis", true, "If set, the agents will watch for the v1beta1 APIs.") flags.BoolVar(&o.EnableClusterInventoryAPIs, "enable-cluster-inventory-apis", false, "If set, the agents will watch for the ClusterInventory APIs.") - flags.DurationVar(&o.ForceDeleteWaitTime.Duration, "force-delete-wait-time", 15*time.Minute, "The duration the hub agent waits before force deleting a member cluster.") + flags.DurationVar(&o.ForceDeleteWaitTime.Duration, "force-delete-wait-time", 5*time.Minute, "The duration the hub agent waits before force deleting a member cluster.") flags.BoolVar(&o.EnableStagedUpdateRunAPIs, "enable-staged-update-run-apis", false, "If set, the agents will watch for the ClusterStagedUpdateRun APIs.") flags.BoolVar(&o.EnableEvictionAPIs, "enable-eviction-apis", false, "If set, the agents will watch for the Eviction and PlacementDisruptionBudget APIs.") diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller.go index a4d8128e3..040fbdf72 100644 --- a/pkg/controllers/membercluster/v1beta1/membercluster_controller.go +++ b/pkg/controllers/membercluster/v1beta1/membercluster_controller.go @@ -141,8 +141,8 @@ func (r *Reconciler) handleDelete(ctx context.Context, mc *clusterv1beta1.Member if !currentNS.DeletionTimestamp.IsZero() { klog.V(2).InfoS("The member cluster namespace is still being deleted", "memberCluster", mcObjRef, "deleteTimestamp", currentNS.DeletionTimestamp) var stuckErr error - if time.Now().After(currentNS.DeletionTimestamp.Add(5 * time.Minute)) { - // alert if the namespace is stuck in deleting for more than 5 minutes + if time.Now().After(currentNS.DeletionTimestamp.Add(10 * time.Minute)) { + // alert if the namespace is stuck in deleting for more than 10 minutes stuckErr = controller.NewUnexpectedBehaviorError(fmt.Errorf("the member cluster namespace %s has been deleting since %s", namespaceName, currentNS.DeletionTimestamp.Format(time.RFC3339))) } return runtime.Result{RequeueAfter: time.Second}, stuckErr diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go index 42220523e..ec0a5646e 100644 --- a/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go +++ b/pkg/controllers/membercluster/v1beta1/membercluster_controller_integration_test.go @@ -59,7 +59,7 @@ var _ = Describe("Test MemberCluster Controller", func() { By("create the member cluster reconciler") r = &Reconciler{ Client: k8sClient, - ForceDeleteWaitTime: 15 * time.Minute, + ForceDeleteWaitTime: 5 * time.Minute, } err := r.SetupWithManager(mgr) Expect(err).Should(Succeed()) @@ -149,7 +149,7 @@ var _ = Describe("Test MemberCluster Controller", func() { result, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: memberClusterNamespacedName, }) - Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute})) + Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime})) Expect(err).Should(Succeed()) var imc clusterv1beta1.InternalMemberCluster @@ -206,7 +206,7 @@ var _ = Describe("Test MemberCluster Controller", func() { By("trigger reconcile again to initiate leave workflow") result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: memberClusterNamespacedName}) - Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute})) + Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime})) Expect(err).Should(Succeed()) var imc clusterv1beta1.InternalMemberCluster @@ -274,7 +274,7 @@ var _ = Describe("Test MemberCluster Controller", func() { r = &Reconciler{ Client: k8sClient, NetworkingAgentsEnabled: true, - ForceDeleteWaitTime: 15 * time.Minute, + ForceDeleteWaitTime: 5 * time.Minute, } err := r.SetupWithManager(mgr) Expect(err).Should(Succeed()) @@ -449,7 +449,7 @@ var _ = Describe("Test MemberCluster Controller", func() { result, err := r.Reconcile(ctx, ctrl.Request{ NamespacedName: memberClusterNamespacedName, }) - Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute})) + Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime})) Expect(err).Should(Succeed()) By("getting imc status") @@ -466,7 +466,7 @@ var _ = Describe("Test MemberCluster Controller", func() { result, err = r.Reconcile(ctx, ctrl.Request{ NamespacedName: memberClusterNamespacedName, }) - Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute})) + Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime})) Expect(err).Should(Succeed()) By("checking mc status") @@ -504,7 +504,7 @@ var _ = Describe("Test MemberCluster Controller", func() { result, err = r.Reconcile(ctx, ctrl.Request{ NamespacedName: memberClusterNamespacedName, }) - Expect(result).Should(Equal(ctrl.Result{RequeueAfter: 15 * time.Minute})) + Expect(result).Should(Equal(ctrl.Result{RequeueAfter: r.ForceDeleteWaitTime})) Expect(err).Should(Succeed()) By("checking mc status")