Use Required Pod anti affinity if Active MDS is not more than 1

When Active MDS is 1, there would be 2 mds pods & we want them to be scheduled on different nodes always. So we use the required pod anti-affinity. If Active MDS are more than 1 then in a 3 node cluster scheduling issues would be there, so we switch to using preferred anti-affinity. Signed-off-by: Malay Kumar Parida <[email protected]>
red-hat-storage · Feb 12, 2025 · e6e5b6b · e6e5b6b
1 parent 377e36d
commit e6e5b6b
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 22 deletions.
diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go
@@ -69,7 +69,7 @@ var (
 				getOcsToleration(),
 			},
 			PodAntiAffinity: &corev1.PodAntiAffinity{
-				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+				RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
 					// left the selector value empty as it will be updated later in the getPlacement()
 				},
 			},

diff --git a/controllers/storagecluster/placement.go b/controllers/storagecluster/placement.go
@@ -22,10 +22,20 @@ func getPlacement(sc *ocsv1.StorageCluster, component string) rookCephv1.Placeme
 		(&in).DeepCopyInto(&placement)
 		// label rook_file_system is added to the mds pod using rook operator
 		if component == "mds" {
-			placement.PodAntiAffinity = &corev1.PodAntiAffinity{
-				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
-					defaults.GetMdsWeightedPodAffinityTerm(100, generateNameForCephFilesystem(sc)),
-				},
+			// if active MDS number is more than 1 then Preferred and if it is 1 then Required pod anti-affinity is set
+			mdsWeightedPodAffinity := defaults.GetMdsWeightedPodAffinityTerm(100, generateNameForCephFilesystem(sc))
+			if sc.Spec.ManagedResources.CephFilesystems.ActiveMetadataServers > 1 {
+				placement.PodAntiAffinity = &corev1.PodAntiAffinity{
+					PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+						mdsWeightedPodAffinity,
+					},
+				}
+			} else {
+				placement.PodAntiAffinity = &corev1.PodAntiAffinity{
+					RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
+						mdsWeightedPodAffinity.PodAffinityTerm,
+					},
+				}
 			}
 		}
 	}

diff --git a/controllers/storagecluster/placement_test.go b/controllers/storagecluster/placement_test.go
@@ -364,21 +364,18 @@ func TestGetPlacement(t *testing.T) {
 					NodeAffinity: defaults.DefaultNodeAffinity,
 					Tolerations:  defaults.DaemonPlacements["mds"].Tolerations,
 					PodAntiAffinity: &corev1.PodAntiAffinity{
-						PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+						RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
 							{
-								Weight: 100,
-								PodAffinityTerm: corev1.PodAffinityTerm{
-									LabelSelector: &metav1.LabelSelector{
-										MatchExpressions: []metav1.LabelSelectorRequirement{
-											{
-												Key:      "rook_file_system",
-												Operator: metav1.LabelSelectorOpIn,
-												Values:   []string{"storage-test-cephfilesystem"},
-											},
+								LabelSelector: &metav1.LabelSelector{
+									MatchExpressions: []metav1.LabelSelectorRequirement{
+										{
+											Key:      "rook_file_system",
+											Operator: metav1.LabelSelectorOpIn,
+											Values:   []string{"storage-test-cephfilesystem"},
 										},
 									},
-									TopologyKey: corev1.LabelZoneFailureDomainStable,
 								},
+								TopologyKey: corev1.LabelZoneFailureDomainStable,
 							},
 						},
 					},
@@ -503,18 +500,18 @@ func TestGetPlacement(t *testing.T) {
 
 		expectedPlacement = c.expectedPlacements["mds"]
 		testPodAffinity := &corev1.PodAntiAffinity{
-			PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
-				defaults.GetMdsWeightedPodAffinityTerm(100, generateNameForCephFilesystem(sc)),
+			RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
+				defaults.GetMdsWeightedPodAffinityTerm(100, generateNameForCephFilesystem(sc)).PodAffinityTerm,
 			},
 		}
 		if expectedPlacement.PodAntiAffinity != nil {
 			topologyKeys := ""
-			if len(expectedPlacement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 0 {
-				topologyKeys = expectedPlacement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.TopologyKey
+			if len(expectedPlacement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) != 0 {
+				topologyKeys = expectedPlacement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey
 			}
-			expectedPlacement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = testPodAffinity.PreferredDuringSchedulingIgnoredDuringExecution
+			expectedPlacement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = testPodAffinity.RequiredDuringSchedulingIgnoredDuringExecution
 			if topologyKeys != "" {
-				expectedPlacement.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.TopologyKey = topologyKeys
+				expectedPlacement.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey = topologyKeys
 			}
 		}
 		actualPlacement = getPlacement(sc, "mds")