Add feature flag to control whether we should add don't evict annotation

pkg/config/config.go

@@ -155,6 +155,9 @@ func complete(ctx context.Context, c *apiv1.Config, getter kclient.Reader, inclu
 	if c.CertManagerIssuer == nil {
 		c.CertManagerIssuer = profile.CertManagerIssuer
 	}
+	if c.AutoConfigureKarpenterDontEvictAnnotations == nil {
+		c.AutoConfigureKarpenterDontEvictAnnotations = profile.AutoConfigureKarpenterDontEvictAnnotations
+	}
 	return nil
 }
 
@@ -453,6 +456,9 @@ func merge(oldConfig, newConfig *apiv1.Config) *apiv1.Config {
 	if newConfig.APIServerCPU != nil {
 		mergedConfig.APIServerCPU = newConfig.APIServerCPU
 	}
+	if newConfig.AutoConfigureKarpenterDontEvictAnnotations != nil {
+		mergedConfig.AutoConfigureKarpenterDontEvictAnnotations = newConfig.AutoConfigureKarpenterDontEvictAnnotations
+	}
 
 	return &mergedConfig
 }

pkg/controller/appdefinition/deploy.go

@@ -806,7 +806,13 @@ func toDeployment(req router.Request, appInstance *v1.AppInstance, tag name.Refe
 
 	// Set karpenter do-not-evict annotation if scale is nil or 1. This prevents karpenter from evicting the pod if deployment is not running with more than 1 replica.
 	if dep.Spec.Replicas == nil || *dep.Spec.Replicas == 1 {
-		dep.Spec.Template.Annotations["karpenter.sh/do-not-evict"] = "true"
+		cfg, err := config.Get(req.Ctx, req.Client)
+		if err != nil {
+			return nil, err
+		}
+		if z.Dereference(cfg.AutoConfigureKarpenterDontEvictAnnotations) {
+			dep.Spec.Template.Annotations["karpenter.sh/do-not-evict"] = "true"
+		}
 	}
 
 	return dep, nil

pkg/profiles/default.go

@@ -73,5 +73,6 @@ func defaultProfile() apiv1.Config {
 		ControllerCPU:                  new(string),
 		APIServerMemory:                new(string),
 		APIServerCPU:                   new(string),
+		AutoConfigureKarpenterDontEvictAnnotations: z.Pointer(true),
 	}
 }

pkg/profiles/production.go

@@ -33,6 +33,7 @@ func productionProfile() apiv1.Config {
 	conf.ControllerCPU = z.Pointer("100m")
 	conf.APIServerMemory = z.Pointer("256Mi")
 	conf.APIServerCPU = z.Pointer("100m")
+	conf.AutoConfigureKarpenterDontEvictAnnotations = z.Pointer(true)
 
 	return conf
 }