Enhanced the affinity for opensearch-cluster to have a default podAnt…

…iAffinity

This PR helps to add a default podAntiAffinity rule in each nodePool
component with the cluster-name as selector to schedule the replicas on
different nodes on Kubernetes.

All components using the same rule, thus each replica from nodePool
components schedules on different nodes. This can be improved on
component wise in nodePool in future if needed.

Fixes #285

- [x] Commits are signed per the DCO using --signoff
- [x] Unittest added for the new/changed functionality and all unit
tests are successful
- [x] Customer-visible features documented
- [x] No linter warnings (`helm lint` and `helm template`)

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and
signing off your commits, please check
[here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

Signed-off-by: Rajesh M <[email protected]>

charts/opensearch-cluster/templates/_helpers.tpl

@@ -58,3 +58,55 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Default pod antiAffinity to nodePool component if no affinity rules defined
+This rule helps the nodePool replicas to schedule on different nodes
+*/}}
+{{- define "nodePools.defaultAffinity" -}}
+affinity:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 100
+      podAffinityTerm:
+        labelSelector:
+          matchLabels:
+            opster.io/opensearch-cluster: {{ $.clusterName }}
+        topologyKey: kubernetes.io/hostname
+{{- end }}
+
+{{/*
+Takes the pod affinity rules from each nodePool and appends the default podAntiAffinity
+*/}}
+{{- define "nodePools.affinity" -}}
+{{- $nodePool := .nodePool -}}
+affinity:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 100
+      podAffinityTerm:
+        labelSelector:
+          matchLabels:
+            opster.io/opensearch-cluster: {{ $.clusterName }}
+        topologyKey: kubernetes.io/hostname
+
+    {{- /* checks if preferredDuringSchedulingIgnoredDuringExecution exists under podAntiAffinity and appending */ -}}
+    {{- if not (empty $nodePool.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution) }}
+    {{ $nodePool.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | toYaml | nindent 4 | trim }}
+    {{- end }}
+
+    {{- /* checks if requiredDuringSchedulingIgnoredDuringExecution exists under podAntiAffinity and appending */ -}}
+    {{- if $nodePool.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution }}
+    requiredDuringSchedulingIgnoredDuringExecution: {{ $nodePool.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | toYaml | nindent 4 }}
+    {{- end }}
+
+  {{- /* checks if podAffinity exists in affinity and appending */ -}}
+  {{- if $nodePool.podAffinity }}
+  podAffinity: {{ $nodePool.podAffinity | toYaml | nindent 4 }}
+  {{- end }}
+
+  {{- /* checks if nodeAffinity exists in affinity and appending */ -}}
+  {{- if $nodePool.nodeAffinity }}
+  nodeAffinity: {{ $nodePool.nodeAffinity | toYaml | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/opensearch-cluster/templates/cluster.yaml

@@ -29,8 +29,16 @@ spec:
   {{- with .Values.cluster.initHelper }}
   initHelper: {{ . | toYaml | nindent 4 }}
   {{- end }}
-  {{- with .Values.cluster.nodePools }}
-  nodePools: {{ . | toYaml | nindent 4 }}
+  nodePools:
+  {{- range .Values.cluster.nodePools }}
+  - {{ omit . "affinity" | toYaml | nindent 4 | trim }}
+    {{- if .affinity }}
+    {{- with .affinity }}
+    {{ include "nodePools.affinity" (dict "nodePool" . "clusterName" $clusterName) | nindent 4 | trim }}
+    {{- end }}
+    {{- else }}
+    {{ include "nodePools.defaultAffinity" (dict "clusterName" $clusterName) | nindent 4 | trim }}
+    {{- end }}
   {{- end }}
   {{- with .Values.cluster.security }}
   security:

charts/opensearch-cluster/values.yaml

@@ -252,6 +252,9 @@ cluster:
         limits:
           memory: "2Gi"
           cpu: "500m"
+      # -- Optional, default podAntiAffinity is added to schedule the nodePool replicas on different nodes
+      # additional affinity rules defined here will be appended with the default podAntiAffinity
+      affinity: {}
 
   # Opensearch security configuration
   security: