diff --git a/api/v1/ocsinitialization_types.go b/api/v1/ocsinitialization_types.go index 5b23ee0f94..91ee500705 100644 --- a/api/v1/ocsinitialization_types.go +++ b/api/v1/ocsinitialization_types.go @@ -54,10 +54,18 @@ type OCSInitializationStatus struct { // operator. Object references will be added to this list after they have // been created AND found in the cluster. // +optional - RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - SCCsCreated bool `json:"sCCsCreated,omitempty"` - RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + SCCsCreated bool `json:"sCCsCreated,omitempty"` + RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RookCephOperatorConfig RookCephOperatorConfigStatus `json:"rookCephOperatorConfig,omitempty"` +} + +type RookCephOperatorConfigStatus struct { + // CsiPluginTolerationsModified indicates if CsiPluginTolerations are added to the configmap via controller + CsiPluginTolerationsModified bool `json:"csiPluginTolerationsModified,omitempty"` + // CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations are added to the configmap via controller + CsiProvisionerTolerationsModified bool `json:"csiProvisionerTolerationsModified,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 55ef5bc1be..621ae5c27c 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -662,6 +662,7 @@ func (in *OCSInitializationStatus) DeepCopyInto(out *OCSInitializationStatus) { *out = make([]corev1.ObjectReference, len(*in)) copy(*out, *in) } + out.RookCephOperatorConfig = in.RookCephOperatorConfig } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCSInitializationStatus. @@ -691,6 +692,21 @@ func (in *OverprovisionControlSpec) DeepCopy() *OverprovisionControlSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RookCephOperatorConfigStatus) DeepCopyInto(out *RookCephOperatorConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RookCephOperatorConfigStatus. +func (in *RookCephOperatorConfigStatus) DeepCopy() *RookCephOperatorConfigStatus { + if in == nil { + return nil + } + out := new(RookCephOperatorConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SharedFilesystemConfigurationSpec) DeepCopyInto(out *SharedFilesystemConfigurationSpec) { *out = *in diff --git a/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml b/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml index ce5313ac6f..2275ed84fc 100644 --- a/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml +++ b/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml @@ -196,6 +196,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go index df4821fa5a..bec7d20b5e 100644 --- a/controllers/defaults/placements.go +++ b/controllers/defaults/placements.go @@ -7,6 +7,11 @@ import ( ) var ( + APIServerKey = "api-server" + MetricsExporterKey = "metrics-exporter" + CsiPluginKey = "csi-plugin" + CsiProvisionerKey = "csi-provisioner" + // osdLabelSelector is the key in OSD pod. Used // as a label selector for topology spread constraints. osdLabelSelector = "rook-ceph-osd" @@ -126,6 +131,30 @@ var ( getOcsToleration(), }, }, + + APIServerKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, + + MetricsExporterKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, + + CsiPluginKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, + + CsiProvisionerKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, } ) diff --git a/controllers/ocsinitialization/ocsinitialization_controller.go b/controllers/ocsinitialization/ocsinitialization_controller.go index e969995f5a..0f62229d70 100644 --- a/controllers/ocsinitialization/ocsinitialization_controller.go +++ b/controllers/ocsinitialization/ocsinitialization_controller.go @@ -9,7 +9,10 @@ import ( "github.com/go-logr/logr" secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -241,7 +244,8 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error { } // ensureRookCephOperatorConfigExists ensures that the rook-ceph-operator-config cm exists -// This configmap is purely reserved for any user overrides to be applied. +// This configmap is semi-reserved for any user overrides to be applied 4.16 onwards. +// Earlier it used to be purely reserved for user overrides. // We don't reconcile it if it exists as it can reset any values the user has set // The configmap is watched by the rook operator and values set here have higher precedence // than the default values set in the rook operator pod env vars. @@ -252,14 +256,75 @@ func (r *OCSInitializationReconciler) ensureRookCephOperatorConfigExists(initial Namespace: initialData.Namespace, }, } - err := r.Client.Create(r.ctx, rookCephOperatorConfig) - if err != nil && !errors.IsAlreadyExists(err) { - r.Log.Error(err, fmt.Sprintf("Failed to create %s configmap", util.RookCephOperatorConfigName)) + + opResult, err := ctrl.CreateOrUpdate(r.ctx, r.Client, rookCephOperatorConfig, func() error { + + if rookCephOperatorConfig.Data == nil { + rookCephOperatorConfig.Data = make(map[string]string) + } + + csiPluginDefaults := defaults.DaemonPlacements[defaults.CsiPluginKey] + csiPluginTolerations := r.getCsiTolerations(defaults.CsiPluginKey) + if err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiPluginTolerations, csiPluginDefaults.Tolerations, "CSI_PLUGIN_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiPluginTolerationsModified); err != nil { + return err + } + + csiProvisionerDefaults := defaults.DaemonPlacements[defaults.CsiProvisionerKey] + csiProvisionerTolerations := r.getCsiTolerations(defaults.CsiProvisionerKey) + if err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiProvisionerTolerations, csiProvisionerDefaults.Tolerations, "CSI_PROVISIONER_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiProvisionerTolerationsModified); err != nil { + return err // nolint:revive + } + + return nil + }) + + if err != nil { + r.Log.Error(err, "Failed to create/update rook-ceph-operator-config configmap") return err } + r.Log.Info("Successfully created/updated rook-ceph-operator-config configmap", "OperationResult", opResult) + return nil } +func updateTolerationsConfigFunc(rookCephOperatorConfig *corev1.ConfigMap, + tolerations, defaults []corev1.Toleration, configMapKey string, modifiedFlag *bool) error { + + if tolerations != nil { + updatedTolerations := append(tolerations, defaults...) + tolerationsYAML, err := yaml.Marshal(updatedTolerations) + if err != nil { + return err + } + rookCephOperatorConfig.Data[configMapKey] = string(tolerationsYAML) + *modifiedFlag = true + } else if tolerations == nil && *modifiedFlag { + delete(rookCephOperatorConfig.Data, configMapKey) + *modifiedFlag = false + } + + return nil +} + +func (r *OCSInitializationReconciler) getCsiTolerations(csiTolerationKey string) []corev1.Toleration { + + var tolerations []corev1.Toleration + + clusters := r.clusters.GetStorageClusters() + + for i := range clusters { + if val, ok := clusters[i].Spec.Placement[rookCephv1.KeyType(csiTolerationKey)]; ok { + tolerations = append(tolerations, val.Tolerations...) + } + } + + return tolerations +} + // ensureOcsOperatorConfigExists ensures that the ocs-operator-config exists & if not create/update it with required values // This configmap is reserved just for ocs operator use, primarily meant for passing values to rook-ceph-operator // It is not meant to be modified by the user diff --git a/controllers/storagecluster/exporter.go b/controllers/storagecluster/exporter.go index 90e95cedd8..fb23762917 100644 --- a/controllers/storagecluster/exporter.go +++ b/controllers/storagecluster/exporter.go @@ -7,6 +7,7 @@ import ( "github.com/imdario/mergo" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/version" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -351,6 +352,7 @@ func deployMetricsExporter(ctx context.Context, r *StorageClusterReconciler, ins }, }, }}, + Tolerations: getPlacement(instance, defaults.MetricsExporterKey).Tolerations, }, } return nil diff --git a/controllers/storagecluster/placement.go b/controllers/storagecluster/placement.go index f11cd44503..fbc90e6380 100644 --- a/controllers/storagecluster/placement.go +++ b/controllers/storagecluster/placement.go @@ -38,6 +38,18 @@ func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placeme return placement } + // if provider-server placements are found in the storagecluster spec append the default ocs tolerations to it + if ok && component == defaults.APIServerKey { + placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...) + return placement + } + + // if metrics-exporter placements are found in the storagecluster spec append the default ocs tolerations to it + if ok && component == defaults.MetricsExporterKey { + placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...) + return placement + } + // If no placement is specified for the given component and the // StorageCluster has no label selector, set the default node // affinity. diff --git a/controllers/storagecluster/provider_server.go b/controllers/storagecluster/provider_server.go index 8516abc57c..3adb22d3b7 100644 --- a/controllers/storagecluster/provider_server.go +++ b/controllers/storagecluster/provider_server.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" "github.com/red-hat-storage/ocs-operator/v4/services/provider/server" ) @@ -379,6 +380,7 @@ func GetProviderAPIServerDeployment(instance *ocsv1.StorageCluster) *appsv1.Depl }, }, }, + Tolerations: getPlacement(instance, defaults.APIServerKey).Tolerations, ServiceAccountName: ocsProviderServerName, }, }, diff --git a/controllers/storagecluster/provider_server_test.go b/controllers/storagecluster/provider_server_test.go index 8520390d2b..418accbe3f 100644 --- a/controllers/storagecluster/provider_server_test.go +++ b/controllers/storagecluster/provider_server_test.go @@ -17,6 +17,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" "github.com/red-hat-storage/ocs-operator/v4/services/provider/server" ) @@ -393,6 +394,7 @@ func GetProviderAPIServerDeploymentForTest(instance *ocsv1.StorageCluster) *apps }, }, ServiceAccountName: ocsProviderServerName, + Tolerations: getPlacement(instance, defaults.APIServerKey).Tolerations, }, }, }, diff --git a/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml b/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml index ce5313ac6f..2275ed84fc 100644 --- a/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml +++ b/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml @@ -196,6 +196,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml b/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml index a658a9ab85..eaeef0c0f5 100644 --- a/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml +++ b/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml @@ -195,6 +195,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/go.mod b/go.mod index 8ed0895cf1..6596d002b0 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( google.golang.org/grpc v1.60.0 google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 + gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.5.0 k8s.io/api v0.28.4 k8s.io/apiextensions-apiserver v0.28.4 @@ -127,7 +128,6 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.28.4 // indirect k8s.io/component-base v0.28.4 // indirect diff --git a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go index 5b23ee0f94..91ee500705 100644 --- a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go +++ b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go @@ -54,10 +54,18 @@ type OCSInitializationStatus struct { // operator. Object references will be added to this list after they have // been created AND found in the cluster. // +optional - RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - SCCsCreated bool `json:"sCCsCreated,omitempty"` - RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + SCCsCreated bool `json:"sCCsCreated,omitempty"` + RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RookCephOperatorConfig RookCephOperatorConfigStatus `json:"rookCephOperatorConfig,omitempty"` +} + +type RookCephOperatorConfigStatus struct { + // CsiPluginTolerationsModified indicates if CsiPluginTolerations are added to the configmap via controller + CsiPluginTolerationsModified bool `json:"csiPluginTolerationsModified,omitempty"` + // CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations are added to the configmap via controller + CsiProvisionerTolerationsModified bool `json:"csiProvisionerTolerationsModified,omitempty"` } // +kubebuilder:object:root=true diff --git a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go index 55ef5bc1be..621ae5c27c 100644 --- a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go +++ b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go @@ -662,6 +662,7 @@ func (in *OCSInitializationStatus) DeepCopyInto(out *OCSInitializationStatus) { *out = make([]corev1.ObjectReference, len(*in)) copy(*out, *in) } + out.RookCephOperatorConfig = in.RookCephOperatorConfig } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCSInitializationStatus. @@ -691,6 +692,21 @@ func (in *OverprovisionControlSpec) DeepCopy() *OverprovisionControlSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RookCephOperatorConfigStatus) DeepCopyInto(out *RookCephOperatorConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RookCephOperatorConfigStatus. +func (in *RookCephOperatorConfigStatus) DeepCopy() *RookCephOperatorConfigStatus { + if in == nil { + return nil + } + out := new(RookCephOperatorConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SharedFilesystemConfigurationSpec) DeepCopyInto(out *SharedFilesystemConfigurationSpec) { *out = *in