From 104f30458ee1a8e453e3f649690c920683647a4b Mon Sep 17 00:00:00 2001 From: Nitin Goyal Date: Wed, 6 Mar 2024 12:18:44 +0530 Subject: [PATCH 1/4] Add mechanism for adding tolerations on provider server Signed-off-by: Nitin Goyal --- controllers/defaults/placements.go | 8 ++++++++ controllers/storagecluster/placement.go | 6 ++++++ controllers/storagecluster/provider_server.go | 2 ++ controllers/storagecluster/provider_server_test.go | 2 ++ 4 files changed, 18 insertions(+) diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go index df4821fa5a..3884bcb546 100644 --- a/controllers/defaults/placements.go +++ b/controllers/defaults/placements.go @@ -7,6 +7,8 @@ import ( ) var ( + APIServerKey = "api-server" + // osdLabelSelector is the key in OSD pod. Used // as a label selector for topology spread constraints. osdLabelSelector = "rook-ceph-osd" @@ -126,6 +128,12 @@ var ( getOcsToleration(), }, }, + + APIServerKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, } ) diff --git a/controllers/storagecluster/placement.go b/controllers/storagecluster/placement.go index f11cd44503..3d959899b1 100644 --- a/controllers/storagecluster/placement.go +++ b/controllers/storagecluster/placement.go @@ -38,6 +38,12 @@ func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placeme return placement } + // if provider-server placements are found in the storagecluster spec append the default ocs tolerations to it + if ok && component == defaults.APIServerKey { + placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...) + return placement + } + // If no placement is specified for the given component and the // StorageCluster has no label selector, set the default node // affinity. diff --git a/controllers/storagecluster/provider_server.go b/controllers/storagecluster/provider_server.go index 8516abc57c..3adb22d3b7 100644 --- a/controllers/storagecluster/provider_server.go +++ b/controllers/storagecluster/provider_server.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" "github.com/red-hat-storage/ocs-operator/v4/services/provider/server" ) @@ -379,6 +380,7 @@ func GetProviderAPIServerDeployment(instance *ocsv1.StorageCluster) *appsv1.Depl }, }, }, + Tolerations: getPlacement(instance, defaults.APIServerKey).Tolerations, ServiceAccountName: ocsProviderServerName, }, }, diff --git a/controllers/storagecluster/provider_server_test.go b/controllers/storagecluster/provider_server_test.go index 8520390d2b..418accbe3f 100644 --- a/controllers/storagecluster/provider_server_test.go +++ b/controllers/storagecluster/provider_server_test.go @@ -17,6 +17,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" "github.com/red-hat-storage/ocs-operator/v4/services/provider/server" ) @@ -393,6 +394,7 @@ func GetProviderAPIServerDeploymentForTest(instance *ocsv1.StorageCluster) *apps }, }, ServiceAccountName: ocsProviderServerName, + Tolerations: getPlacement(instance, defaults.APIServerKey).Tolerations, }, }, }, From 630fdd6afe59eabdd6269ca5e4764cc0c4e7f760 Mon Sep 17 00:00:00 2001 From: Nitin Goyal Date: Wed, 6 Mar 2024 12:38:13 +0530 Subject: [PATCH 2/4] Add mechanism for adding tolerations on metrics exporter Signed-off-by: Nitin Goyal --- controllers/defaults/placements.go | 9 ++++++++- controllers/storagecluster/exporter.go | 2 ++ controllers/storagecluster/placement.go | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go index 3884bcb546..ea79bb8f04 100644 --- a/controllers/defaults/placements.go +++ b/controllers/defaults/placements.go @@ -7,7 +7,8 @@ import ( ) var ( - APIServerKey = "api-server" + APIServerKey = "api-server" + MetricsExporterKey = "metrics-exporter" // osdLabelSelector is the key in OSD pod. Used // as a label selector for topology spread constraints. @@ -134,6 +135,12 @@ var ( getOcsToleration(), }, }, + + MetricsExporterKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, } ) diff --git a/controllers/storagecluster/exporter.go b/controllers/storagecluster/exporter.go index 90e95cedd8..fb23762917 100644 --- a/controllers/storagecluster/exporter.go +++ b/controllers/storagecluster/exporter.go @@ -7,6 +7,7 @@ import ( "github.com/imdario/mergo" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/version" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -351,6 +352,7 @@ func deployMetricsExporter(ctx context.Context, r *StorageClusterReconciler, ins }, }, }}, + Tolerations: getPlacement(instance, defaults.MetricsExporterKey).Tolerations, }, } return nil diff --git a/controllers/storagecluster/placement.go b/controllers/storagecluster/placement.go index 3d959899b1..fbc90e6380 100644 --- a/controllers/storagecluster/placement.go +++ b/controllers/storagecluster/placement.go @@ -44,6 +44,12 @@ func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placeme return placement } + // if metrics-exporter placements are found in the storagecluster spec append the default ocs tolerations to it + if ok && component == defaults.MetricsExporterKey { + placement.Tolerations = append(placement.Tolerations, defaults.DaemonPlacements[component].Tolerations...) + return placement + } + // If no placement is specified for the given component and the // StorageCluster has no label selector, set the default node // affinity. From 83168ddd495198c58ac969f82556ccd7a4cbb2ff Mon Sep 17 00:00:00 2001 From: Nitin Goyal Date: Thu, 7 Mar 2024 17:33:22 +0530 Subject: [PATCH 3/4] Add status field to track toleration modifications Introduce a new status field to track addition of tolerations in the rook-ceph-operator-config configmap. The added field status indicates whether tolerations were successfully added to the configmap. This information is crucial for ensuring safe deletion, as we will only proceed with deletion if the tolerations were added by our mechanism. If they were added by user we will not proceed with the deletion. Signed-off-by: Nitin Goyal --- api/v1/ocsinitialization_types.go | 16 ++++++++++++---- api/v1/zz_generated.deepcopy.go | 16 ++++++++++++++++ .../ocs.openshift.io_ocsinitializations.yaml | 11 +++++++++++ .../ocs/ocs.openshift.io_ocsinitializations.yaml | 11 +++++++++++ .../manifests/ocsinitialization.crd.yaml | 11 +++++++++++ .../api/v4/v1/ocsinitialization_types.go | 16 ++++++++++++---- .../api/v4/v1/zz_generated.deepcopy.go | 16 ++++++++++++++++ 7 files changed, 89 insertions(+), 8 deletions(-) diff --git a/api/v1/ocsinitialization_types.go b/api/v1/ocsinitialization_types.go index 5b23ee0f94..91ee500705 100644 --- a/api/v1/ocsinitialization_types.go +++ b/api/v1/ocsinitialization_types.go @@ -54,10 +54,18 @@ type OCSInitializationStatus struct { // operator. Object references will be added to this list after they have // been created AND found in the cluster. // +optional - RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - SCCsCreated bool `json:"sCCsCreated,omitempty"` - RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + SCCsCreated bool `json:"sCCsCreated,omitempty"` + RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RookCephOperatorConfig RookCephOperatorConfigStatus `json:"rookCephOperatorConfig,omitempty"` +} + +type RookCephOperatorConfigStatus struct { + // CsiPluginTolerationsModified indicates if CsiPluginTolerations are added to the configmap via controller + CsiPluginTolerationsModified bool `json:"csiPluginTolerationsModified,omitempty"` + // CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations are added to the configmap via controller + CsiProvisionerTolerationsModified bool `json:"csiProvisionerTolerationsModified,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 55ef5bc1be..621ae5c27c 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -662,6 +662,7 @@ func (in *OCSInitializationStatus) DeepCopyInto(out *OCSInitializationStatus) { *out = make([]corev1.ObjectReference, len(*in)) copy(*out, *in) } + out.RookCephOperatorConfig = in.RookCephOperatorConfig } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCSInitializationStatus. @@ -691,6 +692,21 @@ func (in *OverprovisionControlSpec) DeepCopy() *OverprovisionControlSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RookCephOperatorConfigStatus) DeepCopyInto(out *RookCephOperatorConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RookCephOperatorConfigStatus. +func (in *RookCephOperatorConfigStatus) DeepCopy() *RookCephOperatorConfigStatus { + if in == nil { + return nil + } + out := new(RookCephOperatorConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SharedFilesystemConfigurationSpec) DeepCopyInto(out *SharedFilesystemConfigurationSpec) { *out = *in diff --git a/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml b/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml index ce5313ac6f..2275ed84fc 100644 --- a/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml +++ b/config/crd/bases/ocs.openshift.io_ocsinitializations.yaml @@ -196,6 +196,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml b/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml index ce5313ac6f..2275ed84fc 100644 --- a/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml +++ b/deploy/csv-templates/crds/ocs/ocs.openshift.io_ocsinitializations.yaml @@ -196,6 +196,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml b/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml index a658a9ab85..eaeef0c0f5 100644 --- a/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml +++ b/deploy/ocs-operator/manifests/ocsinitialization.crd.yaml @@ -195,6 +195,17 @@ spec: type: object x-kubernetes-map-type: atomic type: array + rookCephOperatorConfig: + properties: + csiPluginTolerationsModified: + description: CsiPluginTolerationsModified indicates if CsiPluginTolerations + are added to the configmap via controller + type: boolean + csiProvisionerTolerationsModified: + description: CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations + are added to the configmap via controller + type: boolean + type: object rookCephOperatorConfigCreated: type: boolean sCCsCreated: diff --git a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go index 5b23ee0f94..91ee500705 100644 --- a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go +++ b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/ocsinitialization_types.go @@ -54,10 +54,18 @@ type OCSInitializationStatus struct { // operator. Object references will be added to this list after they have // been created AND found in the cluster. // +optional - RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - SCCsCreated bool `json:"sCCsCreated,omitempty"` - RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RelatedObjects []corev1.ObjectReference `json:"relatedObjects,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + SCCsCreated bool `json:"sCCsCreated,omitempty"` + RookCephOperatorConfigCreated bool `json:"rookCephOperatorConfigCreated,omitempty"` + RookCephOperatorConfig RookCephOperatorConfigStatus `json:"rookCephOperatorConfig,omitempty"` +} + +type RookCephOperatorConfigStatus struct { + // CsiPluginTolerationsModified indicates if CsiPluginTolerations are added to the configmap via controller + CsiPluginTolerationsModified bool `json:"csiPluginTolerationsModified,omitempty"` + // CsiProvisionerTolerationsModified indicates if CsiProvisionerTolerations are added to the configmap via controller + CsiProvisionerTolerationsModified bool `json:"csiProvisionerTolerationsModified,omitempty"` } // +kubebuilder:object:root=true diff --git a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go index 55ef5bc1be..621ae5c27c 100644 --- a/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go +++ b/vendor/github.com/red-hat-storage/ocs-operator/api/v4/v1/zz_generated.deepcopy.go @@ -662,6 +662,7 @@ func (in *OCSInitializationStatus) DeepCopyInto(out *OCSInitializationStatus) { *out = make([]corev1.ObjectReference, len(*in)) copy(*out, *in) } + out.RookCephOperatorConfig = in.RookCephOperatorConfig } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCSInitializationStatus. @@ -691,6 +692,21 @@ func (in *OverprovisionControlSpec) DeepCopy() *OverprovisionControlSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RookCephOperatorConfigStatus) DeepCopyInto(out *RookCephOperatorConfigStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RookCephOperatorConfigStatus. +func (in *RookCephOperatorConfigStatus) DeepCopy() *RookCephOperatorConfigStatus { + if in == nil { + return nil + } + out := new(RookCephOperatorConfigStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SharedFilesystemConfigurationSpec) DeepCopyInto(out *SharedFilesystemConfigurationSpec) { *out = *in From f3f67c2202f7e8f182ef6ede0a7d1d5f65dc7fa1 Mon Sep 17 00:00:00 2001 From: Nitin Goyal Date: Thu, 7 Mar 2024 12:11:43 +0530 Subject: [PATCH 4/4] Add CSI tolerations to rook-ceph-operator-config via controller Enhance the configuration process by incorporating CSI tolerations directly into the rook-ceph-operator-config configmap through the controller. Previously, end users were required to manually edit the configmap for adding CSI tolerations. With this update, users can now seamlessly include CSI tolerations in the storage clusters, and the ocs-operator will automatically propagate these tolerations to the configmap. This improvement simplifies the configuration workflow and ensures that CSI tolerations are seamlessly applied to the Rook Ceph Operator without the need for manual intervention in the configmap. Signed-off-by: Nitin Goyal --- controllers/defaults/placements.go | 14 ++++ .../ocsinitialization_controller.go | 73 ++++++++++++++++++- go.mod | 2 +- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go index ea79bb8f04..bec7d20b5e 100644 --- a/controllers/defaults/placements.go +++ b/controllers/defaults/placements.go @@ -9,6 +9,8 @@ import ( var ( APIServerKey = "api-server" MetricsExporterKey = "metrics-exporter" + CsiPluginKey = "csi-plugin" + CsiProvisionerKey = "csi-provisioner" // osdLabelSelector is the key in OSD pod. Used // as a label selector for topology spread constraints. @@ -141,6 +143,18 @@ var ( getOcsToleration(), }, }, + + CsiPluginKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, + + CsiProvisionerKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, } ) diff --git a/controllers/ocsinitialization/ocsinitialization_controller.go b/controllers/ocsinitialization/ocsinitialization_controller.go index e969995f5a..0f62229d70 100644 --- a/controllers/ocsinitialization/ocsinitialization_controller.go +++ b/controllers/ocsinitialization/ocsinitialization_controller.go @@ -9,7 +9,10 @@ import ( "github.com/go-logr/logr" secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -241,7 +244,8 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error { } // ensureRookCephOperatorConfigExists ensures that the rook-ceph-operator-config cm exists -// This configmap is purely reserved for any user overrides to be applied. +// This configmap is semi-reserved for any user overrides to be applied 4.16 onwards. +// Earlier it used to be purely reserved for user overrides. // We don't reconcile it if it exists as it can reset any values the user has set // The configmap is watched by the rook operator and values set here have higher precedence // than the default values set in the rook operator pod env vars. @@ -252,14 +256,75 @@ func (r *OCSInitializationReconciler) ensureRookCephOperatorConfigExists(initial Namespace: initialData.Namespace, }, } - err := r.Client.Create(r.ctx, rookCephOperatorConfig) - if err != nil && !errors.IsAlreadyExists(err) { - r.Log.Error(err, fmt.Sprintf("Failed to create %s configmap", util.RookCephOperatorConfigName)) + + opResult, err := ctrl.CreateOrUpdate(r.ctx, r.Client, rookCephOperatorConfig, func() error { + + if rookCephOperatorConfig.Data == nil { + rookCephOperatorConfig.Data = make(map[string]string) + } + + csiPluginDefaults := defaults.DaemonPlacements[defaults.CsiPluginKey] + csiPluginTolerations := r.getCsiTolerations(defaults.CsiPluginKey) + if err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiPluginTolerations, csiPluginDefaults.Tolerations, "CSI_PLUGIN_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiPluginTolerationsModified); err != nil { + return err + } + + csiProvisionerDefaults := defaults.DaemonPlacements[defaults.CsiProvisionerKey] + csiProvisionerTolerations := r.getCsiTolerations(defaults.CsiProvisionerKey) + if err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiProvisionerTolerations, csiProvisionerDefaults.Tolerations, "CSI_PROVISIONER_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiProvisionerTolerationsModified); err != nil { + return err // nolint:revive + } + + return nil + }) + + if err != nil { + r.Log.Error(err, "Failed to create/update rook-ceph-operator-config configmap") return err } + r.Log.Info("Successfully created/updated rook-ceph-operator-config configmap", "OperationResult", opResult) + return nil } +func updateTolerationsConfigFunc(rookCephOperatorConfig *corev1.ConfigMap, + tolerations, defaults []corev1.Toleration, configMapKey string, modifiedFlag *bool) error { + + if tolerations != nil { + updatedTolerations := append(tolerations, defaults...) + tolerationsYAML, err := yaml.Marshal(updatedTolerations) + if err != nil { + return err + } + rookCephOperatorConfig.Data[configMapKey] = string(tolerationsYAML) + *modifiedFlag = true + } else if tolerations == nil && *modifiedFlag { + delete(rookCephOperatorConfig.Data, configMapKey) + *modifiedFlag = false + } + + return nil +} + +func (r *OCSInitializationReconciler) getCsiTolerations(csiTolerationKey string) []corev1.Toleration { + + var tolerations []corev1.Toleration + + clusters := r.clusters.GetStorageClusters() + + for i := range clusters { + if val, ok := clusters[i].Spec.Placement[rookCephv1.KeyType(csiTolerationKey)]; ok { + tolerations = append(tolerations, val.Tolerations...) + } + } + + return tolerations +} + // ensureOcsOperatorConfigExists ensures that the ocs-operator-config exists & if not create/update it with required values // This configmap is reserved just for ocs operator use, primarily meant for passing values to rook-ceph-operator // It is not meant to be modified by the user diff --git a/go.mod b/go.mod index 8ed0895cf1..6596d002b0 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( google.golang.org/grpc v1.60.0 google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 + gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.5.0 k8s.io/api v0.28.4 k8s.io/apiextensions-apiserver v0.28.4 @@ -127,7 +128,6 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.28.4 // indirect k8s.io/component-base v0.28.4 // indirect