Merge pull request #2502 from iamniting/tolerations

Apply custom taints via the storagecluster CR

api/v1/ocsinitialization_types.go

@@ -54,10 +54,18 @@ type OCSInitializationStatus struct {
 	// operator. Object references will be added to this list after they have
 	// been created AND found in the cluster.
 	// +optional
-	RelatedObjects                []corev1.ObjectReference `json:"relatedObjects,omitempty"`
-	ErrorMessage                  string                   `json:"errorMessage,omitempty"`
-	SCCsCreated                   bool                     `json:"sCCsCreated,omitempty"`
-	RookCephOperatorConfigCreated bool                     `json:"rookCephOperatorConfigCreated,omitempty"`
+	RelatedObjects                []corev1.ObjectReference     `json:"relatedObjects,omitempty"`
+	ErrorMessage                  string                       `json:"errorMessage,omitempty"`
+	SCCsCreated                   bool                         `json:"sCCsCreated,omitempty"`
+	RookCephOperatorConfigCreated bool                         `json:"rookCephOperatorConfigCreated,omitempty"`
+	RookCephOperatorConfig        RookCephOperatorConfigStatus `json:"rookCephOperatorConfig,omitempty"`
+}
+
+type RookCephOperatorConfigStatus struct {
+	// CsiPluginTolerationsModified indicates if CsiPluginTolerations are added to the configmap via controller
+	CsiPluginTolerationsModified bool `json:"csiPluginTolerationsModified,omitempty"`
+	// CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations are added to the configmap via controller
+	CsiProvisionerTolerationsModified bool `json:"csiProvisionerTolerationsModified,omitempty"`
 }
 
 // +kubebuilder:object:root=true

config/crd/bases/ocs.openshift.io_ocsinitializations.yaml

@@ -196,6 +196,17 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              rookCephOperatorConfig:
+                properties:
+                  csiPluginTolerationsModified:
+                    description: CsiPluginTolerationsModified indicates if CsiPluginTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                  csiProvisionerTolerationsModified:
+                    description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                type: object
               rookCephOperatorConfigCreated:
                 type: boolean
               sCCsCreated:

controllers/defaults/placements.go

@@ -7,6 +7,11 @@ import (
 )
 
 var (
+	APIServerKey       = "api-server"
+	MetricsExporterKey = "metrics-exporter"
+	CsiPluginKey       = "csi-plugin"
+	CsiProvisionerKey  = "csi-provisioner"
+
 	// osdLabelSelector is the key in OSD pod. Used
 	// as a label selector for topology spread constraints.
 	osdLabelSelector = "rook-ceph-osd"
@@ -126,6 +131,30 @@ var (
 				getOcsToleration(),
 			},
 		},
+
+		APIServerKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
+
+		MetricsExporterKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
+
+		CsiPluginKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
+
+		CsiProvisionerKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
 	}
 )
 

controllers/ocsinitialization/ocsinitialization_controller.go

@@ -9,7 +9,10 @@ import (
 	"github.com/go-logr/logr"
 	secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1"
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
+	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -241,7 +244,8 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 // ensureRookCephOperatorConfigExists ensures that the rook-ceph-operator-config cm exists
-// This configmap is purely reserved for any user overrides to be applied.
+// This configmap is semi-reserved for any user overrides to be applied 4.16 onwards.
+// Earlier it used to be purely reserved for user overrides.
 // We don't reconcile it if it exists as it can reset any values the user has set
 // The configmap is watched by the rook operator and values set here have higher precedence
 // than the default values set in the rook operator pod env vars.
@@ -252,14 +256,75 @@ func (r *OCSInitializationReconciler) ensureRookCephOperatorConfigExists(initial
 			Namespace: initialData.Namespace,
 		},
 	}
-	err := r.Client.Create(r.ctx, rookCephOperatorConfig)
-	if err != nil && !errors.IsAlreadyExists(err) {
-		r.Log.Error(err, fmt.Sprintf("Failed to create %s configmap", util.RookCephOperatorConfigName))
+
+	opResult, err := ctrl.CreateOrUpdate(r.ctx, r.Client, rookCephOperatorConfig, func() error {
+
+		if rookCephOperatorConfig.Data == nil {
+			rookCephOperatorConfig.Data = make(map[string]string)
+		}
+
+		csiPluginDefaults := defaults.DaemonPlacements[defaults.CsiPluginKey]
+		csiPluginTolerations := r.getCsiTolerations(defaults.CsiPluginKey)
+		if err := updateTolerationsConfigFunc(rookCephOperatorConfig,
+			csiPluginTolerations, csiPluginDefaults.Tolerations, "CSI_PLUGIN_TOLERATIONS",
+			&initialData.Status.RookCephOperatorConfig.CsiPluginTolerationsModified); err != nil {
+			return err
+		}
+
+		csiProvisionerDefaults := defaults.DaemonPlacements[defaults.CsiProvisionerKey]
+		csiProvisionerTolerations := r.getCsiTolerations(defaults.CsiProvisionerKey)
+		if err := updateTolerationsConfigFunc(rookCephOperatorConfig,
+			csiProvisionerTolerations, csiProvisionerDefaults.Tolerations, "CSI_PROVISIONER_TOLERATIONS",
+			&initialData.Status.RookCephOperatorConfig.CsiProvisionerTolerationsModified); err != nil {
+			return err // nolint:revive
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		r.Log.Error(err, "Failed to create/update rook-ceph-operator-config configmap")
 		return err
 	}
+	r.Log.Info("Successfully created/updated rook-ceph-operator-config configmap", "OperationResult", opResult)
+
 	return nil
 }
 
+func updateTolerationsConfigFunc(rookCephOperatorConfig *corev1.ConfigMap,
+	tolerations, defaults []corev1.Toleration, configMapKey string, modifiedFlag *bool) error {
+
+	if tolerations != nil {
+		updatedTolerations := append(tolerations, defaults...)
+		tolerationsYAML, err := yaml.Marshal(updatedTolerations)
+		if err != nil {
+			return err
+		}
+		rookCephOperatorConfig.Data[configMapKey] = string(tolerationsYAML)
+		*modifiedFlag = true
+	} else if tolerations == nil && *modifiedFlag {
+		delete(rookCephOperatorConfig.Data, configMapKey)
+		*modifiedFlag = false
+	}
+
+	return nil
+}
+
+func (r *OCSInitializationReconciler) getCsiTolerations(csiTolerationKey string) []corev1.Toleration {
+
+	var tolerations []corev1.Toleration
+
+	clusters := r.clusters.GetStorageClusters()
+
+	for i := range clusters {
+		if val, ok := clusters[i].Spec.Placement[rookCephv1.KeyType(csiTolerationKey)]; ok {
+			tolerations = append(tolerations, val.Tolerations...)
+		}
+	}
+
+	return tolerations
+}
+
 // ensureOcsOperatorConfigExists ensures that the ocs-operator-config exists & if not create/update it with required values
 // This configmap is reserved just for ocs operator use, primarily meant for passing values to rook-ceph-operator
 // It is not meant to be modified by the user

controllers/storagecluster/exporter.go

@@ -7,6 +7,7 @@ import (
 	"github.com/imdario/mergo"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/version"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -351,6 +352,7 @@ func deployMetricsExporter(ctx context.Context, r *StorageClusterReconciler, ins
 						},
 					},
 				}},
+				Tolerations: getPlacement(instance, defaults.MetricsExporterKey).Tolerations,
 			},
 		}
 		return nil

controllers/storagecluster/placement.go

@@ -38,6 +38,18 @@ func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placeme
 		return placement
 	}
 
+	// if provider-server placements are found in the storagecluster spec append the default ocs tolerations to it
+	if ok && component == defaults.APIServerKey {
+		placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...)
+		return placement
+	}
+
+	// if metrics-exporter placements are found in the storagecluster spec append the default ocs tolerations to it
+	if ok && component == defaults.MetricsExporterKey {
+		placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...)
+		return placement
+	}
+
 	// If no placement is specified for the given component and the
 	// StorageCluster has no label selector, set the default node
 	// affinity.

controllers/storagecluster/provider_server.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
 	"github.com/red-hat-storage/ocs-operator/v4/services/provider/server"
 )
@@ -379,6 +380,7 @@ func GetProviderAPIServerDeployment(instance *ocsv1.StorageCluster) *appsv1.Depl
 							},
 						},
 					},
+					Tolerations:        getPlacement(instance, defaults.APIServerKey).Tolerations,
 					ServiceAccountName: ocsProviderServerName,
 				},
 			},

controllers/storagecluster/provider_server_test.go

@@ -17,6 +17,7 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
 	"github.com/red-hat-storage/ocs-operator/v4/services/provider/server"
 )
@@ -393,6 +394,7 @@ func GetProviderAPIServerDeploymentForTest(instance *ocsv1.StorageCluster) *apps
 						},
 					},
 					ServiceAccountName: ocsProviderServerName,
+					Tolerations:        getPlacement(instance, defaults.APIServerKey).Tolerations,
 				},
 			},
 		},

deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml

@@ -196,6 +196,17 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              rookCephOperatorConfig:
+                properties:
+                  csiPluginTolerationsModified:
+                    description: CsiPluginTolerationsModified indicates if CsiPluginTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                  csiProvisionerTolerationsModified:
+                    description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                type: object
               rookCephOperatorConfigCreated:
                 type: boolean
               sCCsCreated:

deploy/ocs-operator/manifests/ocsinitialization.crd.yaml

@@ -195,6 +195,17 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              rookCephOperatorConfig:
+                properties:
+                  csiPluginTolerationsModified:
+                    description: CsiPluginTolerationsModified indicates if CsiPluginTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                  csiProvisionerTolerationsModified:
+                    description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations
+                      are added to the configmap via controller
+                    type: boolean
+                type: object
               rookCephOperatorConfigCreated:
                 type: boolean
               sCCsCreated:

go.mod

@@ -36,6 +36,7 @@ require (
 	google.golang.org/grpc v1.60.0
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/ini.v1 v1.67.0
+	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.5.0
 	k8s.io/api v0.28.4
 	k8s.io/apiextensions-apiserver v0.28.4
@@ -127,7 +128,6 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect