csi: add nfs nodeplugin daemonset

This commit adds nfs nodeplugin daemonset. This will allow rook to deploy nfs nodeplugin which can mount/unmount nfs volumes. refer: - https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nfs.md - https://github.com/kubernetes-csi/csi-driver-nfs Signed-off-by: Rakshith R <[email protected]>
Albe83 · Apr 12, 2022 · 34162c3 · 34162c3
1 parent 7ace343
commit 34162c3
Show file tree

Hide file tree

Showing 16 changed files with 358 additions and 18 deletions.
diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md
@@ -106,6 +106,7 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.enableOMAPGenerator`           | EnableOMAP generator deploys omap sidecar in CSI provisioner pod, to enable it set it to true                               | `false`                                                   |
 | `csi.rbdFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted                                  | ReadWriteOnceWithFSType                                   |
 | `csi.cephFSFSGroupPolicy`           | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted                               | ReadWriteOnceWithFSType                                   |
+| `csi.nfsFSGroupPolicy`              | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted                                  | ReadWriteOnceWithFSType                                   |
 | `csi.logLevel`                      | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0`                                                       |
 | `csi.provisionerReplicas`           | Set replicas for csi provisioner deployment.                                                                                | `2`                                                       |
 | `csi.enableGrpcMetrics`             | Enable Ceph CSI GRPC Metrics.                                                                                               | `false`                                                   |
@@ -123,10 +124,13 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.cephFSProvisionerNodeAffinity` | The node labels for affinity of the CephCSI CephFS provisioner deployment (***)                                             | <none>                                                    |
 | `csi.cephFSPluginTolerations`       | Array of tolerations in YAML format which will be added to CephCSI CephFS plugin DaemonSet                                  | <none>                                                    |
 | `csi.cephFSPluginNodeAffinity`      | The node labels for affinity of the CephCSI CephFS plugin DaemonSet (***)                                                   | <none>                                                    |
+| `csi.nfsPluginTolerations`          | Array of tolerations in YAML format which will be added to CephCSI NFS plugin DaemonSet                                     | <none>                                                    |
+| `csi.nfsPluginNodeAffinity`         | The node labels for affinity of the CephCSI NFS plugin DaemonSet (***)                                                      | <none>                                                    |
 | `csi.csiRBDProvisionerResource`     | CEPH CSI RBD provisioner resource requirement list.                                                                         | <none>                                                    |
 | `csi.csiRBDPluginResource`          | CEPH CSI RBD plugin resource requirement list.                                                                              | <none>                                                    |
 | `csi.csiCephFSProvisionerResource`  | CEPH CSI CephFS provisioner resource requirement list.                                                                      | <none>                                                    |
 | `csi.csiCephFSPluginResource`       | CEPH CSI CephFS plugin resource requirement list.                                                                           | <none>                                                    |
+| `csi.csiNFSPluginResource`          | CEPH CSI NFS plugin resource requirement list.                                                                              | <none>                                                    |
 | `csi.cephfsGrpcMetricsPort`         | CSI CephFS driver GRPC metrics port.                                                                                        | `9091`                                                    |
 | `csi.cephfsLivenessMetricsPort`     | CSI CephFS driver metrics port.                                                                                             | `9081`                                                    |
 | `csi.rbdGrpcMetricsPort`            | Ceph CSI RBD driver GRPC metrics port.                                                                                      | `9090`                                                    |
@@ -135,8 +139,9 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.forceCephFSKernelClient`       | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs.                                                | `true`                                                    |
 | `csi.kubeletDirPath`                | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag)                                | `/var/lib/kubelet`                                        |
 | `csi.cephcsi.image`                 | Ceph CSI image.                                                                                                             | `quay.io/cephcsi/cephcsi:v3.6.0`                          |
-| `csi.rbdPluginUpdateStrategy`       | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                                  | `OnDelete`                                                |
-| `csi.cephFSPluginUpdateStrategy`    | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                               | `OnDelete`                                                |
+| `csi.rbdPluginUpdateStrategy`       | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                                  | `RollingUpdate`                                           |
+| `csi.cephFSPluginUpdateStrategy`    | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                               | `RollingUpdate`                                           |
+| `csi.nfsPluginUpdateStrategy`       | CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                                  | `RollingUpdate`                                           |
 | `csi.registrar.image`               | Kubernetes CSI registrar image.                                                                                             | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0` |
 | `csi.resizer.image`                 | Kubernetes CSI resizer image.                                                                                               | `k8s.gcr.io/sig-storage/csi-resizer:v1.4.0`               |
 | `csi.provisioner.image`             | Kubernetes CSI provisioner image.                                                                                           | `k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0`           |
@@ -148,6 +153,8 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.volumeReplication.image`       | Volume Replication Controller image.                                                                                        | `quay.io/csiaddons/volumereplication-operator:v0.3.0`     |
 | `csi.csiAddons.enabled`     | Enable CSIAddons                                                                                                  | `false`                                                   |
 | `csi.csiAddons.image`       | CSIAddons Sidecar image.                                                                                        | `quay.io/csiaddons/k8s-sidecar:v0.2.1`     |
+| `csi.nfs.enabled`                   | Enable nfs driver.                                                                                                          | `false`                                                   |
+| `csi.nfs.image`                     | NFS nodeplugin image.                                                                                                       | `mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0`                |
 | `admissionController.tolerations`   | Array of tolerations in YAML format which will be added to admission controller deployment.                                 | <none>                                                    |
 | `admissionController.nodeAffinity`  | The node labels for affinity of the admission controller deployment (***)                                                   | <none>                                                    |
 | `monitoring.enabled`                | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` |

diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh
@@ -19,6 +19,7 @@ options=(
   --namespace rook-ceph
   --set crds.enabled=false
   --set csi.csiAddons.enabled=true
+  --set csi.nfs.enabled=true
 )
 if [[ -z "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES}" ]]; then
   options+=(--set pspEnable=true)

diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml
@@ -27,12 +27,18 @@ data:
 {{- if .Values.csi.cephFSPluginUpdateStrategy }}
   CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }}
 {{- end }}
+{{- if .Values.csi.nfsPluginUpdateStrategy }}
+  CSI_NFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.nfsPluginUpdateStrategy | quote }}
+{{- end }}
 {{- if .Values.csi.rbdFSGroupPolicy }}
   CSI_RBD_FSGROUPPOLICY: {{ .Values.csi.rbdFSGroupPolicy | quote }}
 {{- end }}
 {{- if .Values.csi.cephFSFSGroupPolicy }}
   CSI_CEPHFS_FSGROUPPOLICY: {{ .Values.csi.cephFSFSGroupPolicy | quote }}
 {{- end }}
+{{- if .Values.csi.nfsFSGroupPolicy }}
+  CSI_NFS_FSGROUPPOLICY: {{ .Values.csi.nfsFSGroupPolicy | quote }}
+{{- end }}
 {{- if .Values.csi.rbdPluginUpdateStrategy }}
   CSI_RBD_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.rbdPluginUpdateStrategy | quote }}
 {{- end }}
@@ -82,9 +88,18 @@ data:
   ROOK_CSIADDONS_IMAGE: {{ .Values.csi.csiAddons.image | quote }}
 {{- end }}
 {{- end }}
+{{- if .Values.csi.nfs }}
+  ROOK_CSI_ENABLE_NFS: {{ .Values.csi.nfs.enabled | quote }}
+{{- if .Values.csi.nfs.image }}
+  ROOK_CSI_NFS_IMAGE: {{ .Values.csi.nfs.image | quote }}
+{{- end }}
+{{- end }}
 {{- if .Values.csi.cephfsPodLabels }}
   ROOK_CSI_CEPHFS_POD_LABELS: {{ .Values.csi.cephfsPodLabels | quote }}
 {{- end }}
+{{- if .Values.csi.nfsPodLabels }}
+  ROOK_CSI_NFS_POD_LABELS: {{ .Values.csi.nfsPodLabels | quote }}
+{{- end }}
 {{- if .Values.csi.rbdPodLabels }}
   ROOK_CSI_RBD_POD_LABELS: {{ .Values.csi.rbdPodLabels | quote }}
 {{- end }}
@@ -127,6 +142,12 @@ data:
 {{- if .Values.csi.cephFSPluginNodeAffinity }}
   CSI_CEPHFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.cephFSPluginNodeAffinity }}
 {{- end }}
+{{- if .Values.csi.nfsPluginTolerations }}
+  CSI_NFS_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.nfsPluginTolerations | quote }}
+{{- end }}
+{{- if .Values.csi.nfsPluginNodeAffinity }}
+  CSI_NFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.nfsPluginNodeAffinity }}
+{{- end }}
 {{- if .Values.csi.cephfsGrpcMetricsPort }}
   CSI_CEPHFS_GRPC_METRICS_PORT: {{ .Values.csi.cephfsGrpcMetricsPort | quote }}
 {{- end }}
@@ -160,4 +181,7 @@ data:
 {{- if .Values.csi.csiCephFSPluginResource }}
   CSI_CEPHFS_PLUGIN_RESOURCE: {{ .Values.csi.csiCephFSPluginResource | quote }}
 {{- end }}
+{{- if .Values.csi.csiNFSPluginResource }}
+  CSI_NFS_PLUGIN_RESOURCE: {{ .Values.csi.csiNFSPluginResource | quote }}
+{{- end }}
 {{- end }}
diff --git a/deploy/charts/rook-ceph/templates/serviceaccount.yaml b/deploy/charts/rook-ceph/templates/serviceaccount.yaml
@@ -18,6 +18,16 @@ metadata:
   namespace: {{ .Release.Namespace }} # namespace:operator
 {{ template "library.imagePullSecrets" . }}
 ---
+{{- if .Values.csi.nfs.enabled }}
+# Service account for the NFS CSI driver
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rook-csi-nfs-plugin-sa
+  namespace: {{ .Release.Namespace }} # namespace:operator
+{{ template "library.imagePullSecrets" . }}
+---
+{{ end }}
 # Service account for the CephFS CSI provisioner
 apiVersion: v1
 kind: ServiceAccount

diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
@@ -83,6 +83,10 @@ csi:
   # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
   # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
   cephFSFSGroupPolicy: "ReadWriteOnceWithFSType"
+
+  # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted.
+  # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
+  nfsFSGroupPolicy: "ReadWriteOnceWithFSType"
 
   # OMAP generator generates the omap mapping between the PV name and the RBD image
   # which helps CSI to identify the rbd images for CSI operations.
@@ -97,12 +101,16 @@ csi:
   # Set logging level for csi containers.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   #logLevel: 0
-  # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
+  # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
   # Default value is RollingUpdate.
   #rbdPluginUpdateStrategy: OnDelete
-  # CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
+  # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
   # Default value is RollingUpdate.
   #cephFSPluginUpdateStrategy: OnDelete
+  # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
+  # Default value is RollingUpdate.
+  #nfsPluginUpdateStrategy: OnDelete
+
   # Allow starting unsupported ceph-csi image
   allowUnsupportedVersion: false
     # CEPH CSI RBD provisioner resource requirement list, Put here list of resource
@@ -253,6 +261,26 @@ csi:
         limits:
           memory: 256Mi
           cpu: 100m
+  # CEPH CSI NFS plugin resource requirement list, Put here list of resource
+  # requests and limits you want to apply for plugin pod
+  csiNFSPluginResource: |
+   - name : driver-registrar
+     resource:
+       requests:
+         memory: 128Mi
+         cpu: 50m
+       limits:
+         memory: 256Mi
+         cpu: 100m
+   - name : csi-nfsplugin
+     resource:
+       requests:
+         memory: 512Mi
+         cpu: 250m
+       limits:
+         memory: 1Gi
+         cpu: 500m
+
   # Set provisonerTolerations and provisionerNodeAffinity for provisioner pod.
   # The CSI provisioner would be best to start on the same nodes as other ceph daemons.
   # provisionerTolerations:
@@ -291,6 +319,8 @@ csi:
     #image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
   # Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
   #cephfsPodLabels: "key1=value1,key2=value2"
+  # Labels to add to the CSI NFS Deployments and DaemonSets Pods.
+  #nfsPodLabels: "key1=value1,key2=value2"
   # Labels to add to the CSI RBD Deployments and DaemonSets Pods.
   #rbdPodLabels: "key1=value1,key2=value2"
   # Enable the volume replication controller.
@@ -303,6 +333,10 @@ csi:
   csiAddons:
     enabled: false
     #image: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
+  # Enable the nfs csi driver.
+  nfs:
+    enabled: false
+    #image: "mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0"
 enableDiscoveryDaemon: false
 cephCommandsTimeoutSeconds: "15"
 

diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml
@@ -1362,6 +1362,15 @@ metadata:
 # imagePullSecrets:
 #   - name: my-registry-secret
 ---
+# Service account for the NFS CSI driver
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rook-csi-nfs-plugin-sa
+  namespace: rook-ceph # namespace:operator
+# imagePullSecrets:
+#   - name: my-registry-secret
+---
 # Service account for the RBD CSI driver
 apiVersion: v1
 kind: ServiceAccount

diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
@@ -3,6 +3,7 @@
  k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
  k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
  k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
+ mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0
  quay.io/ceph/ceph:v16.2.7
  quay.io/cephcsi/cephcsi:v3.6.0
  quay.io/csiaddons/k8s-sidecar:v0.2.1