From 34162c3dc58aca3897b76e66f672d62592414497 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Wed, 16 Mar 2022 11:23:37 +0530 Subject: [PATCH] csi: add nfs nodeplugin daemonset This commit adds nfs nodeplugin daemonset. This will allow rook to deploy nfs nodeplugin which can mount/unmount nfs volumes. refer: - https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nfs.md - https://github.com/kubernetes-csi/csi-driver-nfs Signed-off-by: Rakshith R --- Documentation/helm-operator.md | 11 ++- build/rbac/get-helm-rbac.sh | 1 + .../charts/rook-ceph/templates/configmap.yaml | 24 +++++ .../rook-ceph/templates/serviceaccount.yaml | 10 +++ deploy/charts/rook-ceph/values.yaml | 38 +++++++- deploy/examples/common.yaml | 9 ++ deploy/examples/images.txt | 1 + deploy/examples/operator-openshift.yaml | 43 +++++++++ deploy/examples/operator.yaml | 42 +++++++++ images/ceph/Makefile | 2 +- pkg/operator/ceph/csi/betav1csidriver.go | 5 +- pkg/operator/ceph/csi/controller.go | 4 +- pkg/operator/ceph/csi/csi.go | 7 ++ pkg/operator/ceph/csi/csidriver.go | 7 +- pkg/operator/ceph/csi/spec.go | 82 ++++++++++++++++- .../ceph/csi/template/nfs/csi-nfsplugin.yaml | 90 +++++++++++++++++++ 16 files changed, 358 insertions(+), 18 deletions(-) create mode 100644 pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml diff --git a/Documentation/helm-operator.md b/Documentation/helm-operator.md index 6fea094ccef0..b3ecea8454ab 100644 --- a/Documentation/helm-operator.md +++ b/Documentation/helm-operator.md @@ -106,6 +106,7 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.enableOMAPGenerator` | EnableOMAP generator deploys omap sidecar in CSI provisioner pod, to enable it set it to true | `false` | | `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted | ReadWriteOnceWithFSType | | `csi.cephFSFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted | ReadWriteOnceWithFSType | +| `csi.nfsFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted | ReadWriteOnceWithFSType | | `csi.logLevel` | Set logging level for csi containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. | `0` | | `csi.provisionerReplicas` | Set replicas for csi provisioner deployment. | `2` | | `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics. | `false` | @@ -123,10 +124,13 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.cephFSProvisionerNodeAffinity` | The node labels for affinity of the CephCSI CephFS provisioner deployment (***) | | | `csi.cephFSPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI CephFS plugin DaemonSet | | | `csi.cephFSPluginNodeAffinity` | The node labels for affinity of the CephCSI CephFS plugin DaemonSet (***) | | +| `csi.nfsPluginTolerations` | Array of tolerations in YAML format which will be added to CephCSI NFS plugin DaemonSet | | +| `csi.nfsPluginNodeAffinity` | The node labels for affinity of the CephCSI NFS plugin DaemonSet (***) | | | `csi.csiRBDProvisionerResource` | CEPH CSI RBD provisioner resource requirement list. | | | `csi.csiRBDPluginResource` | CEPH CSI RBD plugin resource requirement list. | | | `csi.csiCephFSProvisionerResource` | CEPH CSI CephFS provisioner resource requirement list. | | | `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list. | | +| `csi.csiNFSPluginResource` | CEPH CSI NFS plugin resource requirement list. | | | `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port. | `9091` | | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port. | `9081` | | `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` | @@ -135,8 +139,9 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` | | `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` | | `csi.cephcsi.image` | Ceph CSI image. | `quay.io/cephcsi/cephcsi:v3.6.0` | -| `csi.rbdPluginUpdateStrategy` | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `OnDelete` | -| `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `OnDelete` | +| `csi.rbdPluginUpdateStrategy` | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | +| `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | +| `csi.nfsPluginUpdateStrategy` | CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. | `RollingUpdate` | | `csi.registrar.image` | Kubernetes CSI registrar image. | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0` | | `csi.resizer.image` | Kubernetes CSI resizer image. | `k8s.gcr.io/sig-storage/csi-resizer:v1.4.0` | | `csi.provisioner.image` | Kubernetes CSI provisioner image. | `k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0` | @@ -148,6 +153,8 @@ The following tables lists the configurable parameters of the rook-operator char | `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.3.0` | | `csi.csiAddons.enabled` | Enable CSIAddons | `false` | | `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.2.1` | +| `csi.nfs.enabled` | Enable nfs driver. | `false` | +| `csi.nfs.image` | NFS nodeplugin image. | `mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0` | | `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | | | `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment (***) | | | `monitoring.enabled` | Create necessary RBAC rules for Rook to integrate with Prometheus monitoring in the operator namespace. Requires Prometheus to be pre-installed. | `false` | diff --git a/build/rbac/get-helm-rbac.sh b/build/rbac/get-helm-rbac.sh index bdce10de3fb9..af758072f665 100755 --- a/build/rbac/get-helm-rbac.sh +++ b/build/rbac/get-helm-rbac.sh @@ -19,6 +19,7 @@ options=( --namespace rook-ceph --set crds.enabled=false --set csi.csiAddons.enabled=true + --set csi.nfs.enabled=true ) if [[ -z "${DO_NOT_INCLUDE_POD_SECURITY_POLICY_RESOURCES}" ]]; then options+=(--set pspEnable=true) diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index ccc0a3a48468..3760379681d7 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -27,12 +27,18 @@ data: {{- if .Values.csi.cephFSPluginUpdateStrategy }} CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.cephFSPluginUpdateStrategy | quote }} {{- end }} +{{- if .Values.csi.nfsPluginUpdateStrategy }} + CSI_NFS_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.nfsPluginUpdateStrategy | quote }} +{{- end }} {{- if .Values.csi.rbdFSGroupPolicy }} CSI_RBD_FSGROUPPOLICY: {{ .Values.csi.rbdFSGroupPolicy | quote }} {{- end }} {{- if .Values.csi.cephFSFSGroupPolicy }} CSI_CEPHFS_FSGROUPPOLICY: {{ .Values.csi.cephFSFSGroupPolicy | quote }} {{- end }} +{{- if .Values.csi.nfsFSGroupPolicy }} + CSI_NFS_FSGROUPPOLICY: {{ .Values.csi.nfsFSGroupPolicy | quote }} +{{- end }} {{- if .Values.csi.rbdPluginUpdateStrategy }} CSI_RBD_PLUGIN_UPDATE_STRATEGY: {{ .Values.csi.rbdPluginUpdateStrategy | quote }} {{- end }} @@ -82,9 +88,18 @@ data: ROOK_CSIADDONS_IMAGE: {{ .Values.csi.csiAddons.image | quote }} {{- end }} {{- end }} +{{- if .Values.csi.nfs }} + ROOK_CSI_ENABLE_NFS: {{ .Values.csi.nfs.enabled | quote }} +{{- if .Values.csi.nfs.image }} + ROOK_CSI_NFS_IMAGE: {{ .Values.csi.nfs.image | quote }} +{{- end }} +{{- end }} {{- if .Values.csi.cephfsPodLabels }} ROOK_CSI_CEPHFS_POD_LABELS: {{ .Values.csi.cephfsPodLabels | quote }} {{- end }} +{{- if .Values.csi.nfsPodLabels }} + ROOK_CSI_NFS_POD_LABELS: {{ .Values.csi.nfsPodLabels | quote }} +{{- end }} {{- if .Values.csi.rbdPodLabels }} ROOK_CSI_RBD_POD_LABELS: {{ .Values.csi.rbdPodLabels | quote }} {{- end }} @@ -127,6 +142,12 @@ data: {{- if .Values.csi.cephFSPluginNodeAffinity }} CSI_CEPHFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.cephFSPluginNodeAffinity }} {{- end }} +{{- if .Values.csi.nfsPluginTolerations }} + CSI_NFS_PLUGIN_TOLERATIONS: {{ toYaml .Values.csi.nfsPluginTolerations | quote }} +{{- end }} +{{- if .Values.csi.nfsPluginNodeAffinity }} + CSI_NFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.nfsPluginNodeAffinity }} +{{- end }} {{- if .Values.csi.cephfsGrpcMetricsPort }} CSI_CEPHFS_GRPC_METRICS_PORT: {{ .Values.csi.cephfsGrpcMetricsPort | quote }} {{- end }} @@ -160,4 +181,7 @@ data: {{- if .Values.csi.csiCephFSPluginResource }} CSI_CEPHFS_PLUGIN_RESOURCE: {{ .Values.csi.csiCephFSPluginResource | quote }} {{- end }} +{{- if .Values.csi.csiNFSPluginResource }} + CSI_NFS_PLUGIN_RESOURCE: {{ .Values.csi.csiNFSPluginResource | quote }} +{{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/serviceaccount.yaml b/deploy/charts/rook-ceph/templates/serviceaccount.yaml index aceeb2a0a6de..ba4fe7e11353 100644 --- a/deploy/charts/rook-ceph/templates/serviceaccount.yaml +++ b/deploy/charts/rook-ceph/templates/serviceaccount.yaml @@ -18,6 +18,16 @@ metadata: namespace: {{ .Release.Namespace }} # namespace:operator {{ template "library.imagePullSecrets" . }} --- +{{- if .Values.csi.nfs.enabled }} +# Service account for the NFS CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-nfs-plugin-sa + namespace: {{ .Release.Namespace }} # namespace:operator +{{ template "library.imagePullSecrets" . }} +--- +{{ end }} # Service account for the CephFS CSI provisioner apiVersion: v1 kind: ServiceAccount diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 4f51285f7a29..8cdca808079d 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -83,6 +83,10 @@ csi: # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html cephFSFSGroupPolicy: "ReadWriteOnceWithFSType" + + # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. + # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html + nfsFSGroupPolicy: "ReadWriteOnceWithFSType" # OMAP generator generates the omap mapping between the PV name and the RBD image # which helps CSI to identify the rbd images for CSI operations. @@ -97,12 +101,16 @@ csi: # Set logging level for csi containers. # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. #logLevel: 0 - # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. + # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. #rbdPluginUpdateStrategy: OnDelete - # CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. + # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. # Default value is RollingUpdate. #cephFSPluginUpdateStrategy: OnDelete + # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. + # Default value is RollingUpdate. + #nfsPluginUpdateStrategy: OnDelete + # Allow starting unsupported ceph-csi image allowUnsupportedVersion: false # CEPH CSI RBD provisioner resource requirement list, Put here list of resource @@ -253,6 +261,26 @@ csi: limits: memory: 256Mi cpu: 100m + # CEPH CSI NFS plugin resource requirement list, Put here list of resource + # requests and limits you want to apply for plugin pod + csiNFSPluginResource: | + - name : driver-registrar + resource: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 100m + - name : csi-nfsplugin + resource: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 500m + # Set provisonerTolerations and provisionerNodeAffinity for provisioner pod. # The CSI provisioner would be best to start on the same nodes as other ceph daemons. # provisionerTolerations: @@ -291,6 +319,8 @@ csi: #image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 # Labels to add to the CSI CephFS Deployments and DaemonSets Pods. #cephfsPodLabels: "key1=value1,key2=value2" + # Labels to add to the CSI NFS Deployments and DaemonSets Pods. + #nfsPodLabels: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. #rbdPodLabels: "key1=value1,key2=value2" # Enable the volume replication controller. @@ -303,6 +333,10 @@ csi: csiAddons: enabled: false #image: "quay.io/csiaddons/k8s-sidecar:v0.2.1" + # Enable the nfs csi driver. + nfs: + enabled: false + #image: "mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0" enableDiscoveryDaemon: false cephCommandsTimeoutSeconds: "15" diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index 9f1f7691a1b4..13ec97e74ce9 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -1362,6 +1362,15 @@ metadata: # imagePullSecrets: # - name: my-registry-secret --- +# Service account for the NFS CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-nfs-plugin-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- # Service account for the RBD CSI driver apiVersion: v1 kind: ServiceAccount diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt index 30395c5fd0f6..9f237f87266f 100644 --- a/deploy/examples/images.txt +++ b/deploy/examples/images.txt @@ -3,6 +3,7 @@ k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1 + mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0 quay.io/ceph/ceph:v16.2.7 quay.io/cephcsi/cephcsi:v3.6.0 quay.io/csiaddons/k8s-sidecar:v0.2.1 diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index b011bf689f46..cf5efc59da32 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -93,6 +93,7 @@ users: - system:serviceaccount:rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator + - system:serviceaccount:rook-ceph:rook-csi-nfs-plugin-sa # serviceaccount:namespace:operator --- # Rook Ceph Operator Config # Use this ConfigMap to override operator configurations @@ -113,6 +114,8 @@ data: ROOK_CSI_ENABLE_CEPHFS: "true" # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_RBD: "true" + # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. + ROOK_CSI_ENABLE_NFS: "false" ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary @@ -153,6 +156,10 @@ data: # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType" + # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. + # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html + CSI_NFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType" + # (Optional) Allow starting unsupported ceph-csi image ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false" @@ -168,6 +175,7 @@ data: # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0" # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1" # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0" + # ROOK_CSI_NFS_IMAGE: "mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0" # (Optional) set user created priorityclassName for csi plugin pods. # CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical" @@ -182,6 +190,10 @@ data: # Default value is RollingUpdate. # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. + # Default value is RollingUpdate. + # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet" @@ -189,6 +201,8 @@ data: # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2" + # Labels to add to the CSI NFS Deployments and DaemonSets Pods. + # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" # (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner). # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" @@ -249,6 +263,15 @@ data: # - key: node.rook.io/cephfs # operator: Exists + # (Optional) CephCSI NFS plugin NodeAffinity (overrides CSI_PLUGIN_NODE_AFFINITY). + # CSI_NFS_PLUGIN_NODE_AFFINITY: "role=nfs-node" + # (Optional) CephCSI NFS plugin tolerations list (overrides CSI_PLUGIN_TOLERATIONS). + # Put here list of taints you want to tolerate in YAML format. + # CSI plugins need to be started on all the nodes where the clients need to mount the storage. + # CSI_NFS_PLUGIN_TOLERATIONS: | + # - key: node.rook.io/nfs + # operator: Exists + # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource # requests and limits you want to apply for provisioner pod # CSI_RBD_PROVISIONER_RESOURCE: | @@ -398,6 +421,26 @@ data: # memory: 256Mi # cpu: 100m + # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource + # requests and limits you want to apply for plugin pod + # CSI_NFS_PLUGIN_RESOURCE: | + # - name : driver-registrar + # resource: + # requests: + # memory: 128Mi + # cpu: 50m + # limits: + # memory: 256Mi + # cpu: 100m + # - name : csi-nfsplugin + # resource: + # requests: + # memory: 512Mi + # cpu: 250m + # limits: + # memory: 1Gi + # cpu: 500m + # Configure CSI Ceph FS grpc and liveness metrics port # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 21a5a1276289..d9a178254a81 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -30,6 +30,8 @@ data: ROOK_CSI_ENABLE_CEPHFS: "true" # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_RBD: "true" + # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. + ROOK_CSI_ENABLE_NFS: "false" ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary @@ -70,6 +72,10 @@ data: # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType" + # (Optional) policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. + # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html + CSI_NFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType" + # (Optional) Allow starting unsupported ceph-csi image ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false" @@ -85,6 +91,7 @@ data: # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0" # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1" # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0" + # ROOK_CSI_NFS_IMAGE: "mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0" # (Optional) set user created priorityclassName for csi plugin pods. # CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical" @@ -99,6 +106,10 @@ data: # Default value is RollingUpdate. # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate. + # Default value is RollingUpdate. + # CSI_NFS_PLUGIN_UPDATE_STRATEGY: "OnDelete" + # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path. # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet" @@ -106,6 +117,8 @@ data: # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2" # Labels to add to the CSI RBD Deployments and DaemonSets Pods. # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2" + # Labels to add to the CSI NFS Deployments and DaemonSets Pods. + # ROOK_CSI_NFS_POD_LABELS: "key1=value1,key2=value2" # (Optional) CephCSI provisioner NodeAffinity(applied to both CephFS and RBD provisioner). # CSI_PROVISIONER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" @@ -166,6 +179,15 @@ data: # - key: node.rook.io/cephfs # operator: Exists + # (Optional) CephCSI NFS plugin NodeAffinity (overrides CSI_PLUGIN_NODE_AFFINITY). + # CSI_NFS_PLUGIN_NODE_AFFINITY: "role=nfs-node" + # (Optional) CephCSI NFS plugin tolerations list (overrides CSI_PLUGIN_TOLERATIONS). + # Put here list of taints you want to tolerate in YAML format. + # CSI plugins need to be started on all the nodes where the clients need to mount the storage. + # CSI_NFS_PLUGIN_TOLERATIONS: | + # - key: node.rook.io/nfs + # operator: Exists + # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource # requests and limits you want to apply for provisioner pod # CSI_RBD_PROVISIONER_RESOURCE: | @@ -315,6 +337,26 @@ data: # memory: 256Mi # cpu: 100m + # (Optional) CEPH CSI NFS plugin resource requirement list, Put here list of resource + # requests and limits you want to apply for plugin pod + # CSI_NFS_PLUGIN_RESOURCE: | + # - name : driver-registrar + # resource: + # requests: + # memory: 128Mi + # cpu: 50m + # limits: + # memory: 256Mi + # cpu: 100m + # - name : csi-nfsplugin + # resource: + # requests: + # memory: 512Mi + # cpu: 250m + # limits: + # memory: 1Gi + # cpu: 500m + # Configure CSI CSI Ceph FS grpc and liveness metrics port # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 6e3738c4d504..68a3929b3d50 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -147,7 +147,7 @@ list-image: ## Create a list of images for offline installation rm -f $(IMAGE_TMP) awk '/image:/ {print $2}' $(MANIFESTS_DIR)/operator.yaml $(MANIFESTS_DIR)/cluster.yaml | \ cut -d: -f2- | tee $(IMAGE_TMP) - awk '/quay.io/ || /k8s.gcr.io/ {print $3}' ../../pkg/operator/ceph/csi/spec.go | \ + awk '/quay.io/ || /k8s.gcr.io/ || /mcr.microsoft.com/ {print $3}' ../../pkg/operator/ceph/csi/spec.go | \ cut -d= -f2- | tr -d '"' | tee -a $(IMAGE_TMP) rm -f $(MANIFESTS_DIR)/images.txt cat $(IMAGE_TMP) | sort -h | uniq | tee $(MANIFESTS_DIR)/images.txt diff --git a/pkg/operator/ceph/csi/betav1csidriver.go b/pkg/operator/ceph/csi/betav1csidriver.go index ec3996e0c428..efbf4c77fb7a 100644 --- a/pkg/operator/ceph/csi/betav1csidriver.go +++ b/pkg/operator/ceph/csi/betav1csidriver.go @@ -39,8 +39,7 @@ type beta1CsiDriver struct { } // createCSIDriverInfo Registers CSI driver by creating a CSIDriver object -func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error { - attach := true +func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string, attachRequired bool) error { mountInfo := false // Create CSIDriver object csiDriver := &betav1k8scsi.CSIDriver{ @@ -48,7 +47,7 @@ func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kuber Name: name, }, Spec: betav1k8scsi.CSIDriverSpec{ - AttachRequired: &attach, + AttachRequired: &attachRequired, PodInfoOnMount: &mountInfo, }, } diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index 13e990898c7f..44b334d48d7e 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -117,7 +117,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e if err != nil { if kerrors.IsNotFound(err) { logger.Debug("no ceph cluster found not deploying ceph csi driver") - EnableRBD, EnableCephFS = false, false + EnableRBD, EnableCephFS, EnableNFS = false, false, false err = r.stopDrivers(serverVersion) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") @@ -132,7 +132,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e // // Do not nothing if no ceph cluster is present if len(cephClusters.Items) == 0 { logger.Debug("no ceph cluster found not deploying ceph csi driver") - EnableRBD, EnableCephFS = false, false + EnableRBD, EnableCephFS, EnableNFS = false, false, false err = r.stopDrivers(serverVersion) if err != nil { return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers") diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 726c48b87f9f..9b9bc50aa025 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -67,6 +67,10 @@ func (r *ReconcileCSI) setParams() error { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'") } + if EnableNFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_NFS", "false")); err != nil { + return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_NFS'") + } + if AllowUnsupported, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ALLOW_UNSUPPORTED_VERSION", "false")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'") } @@ -80,6 +84,7 @@ func (r *ReconcileCSI) setParams() error { } CSIParam.CSIPluginImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) + CSIParam.NFSPluginImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_NFS_IMAGE", DefaultNFSPluginImage) CSIParam.RegistrarImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) CSIParam.ProvisionerImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) CSIParam.AttacherImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) @@ -89,6 +94,8 @@ func (r *ReconcileCSI) setParams() error { CSIParam.CSIAddonsImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "") CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) + csiNFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_NFS_POD_LABELS", "") + CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels) csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "") CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) diff --git a/pkg/operator/ceph/csi/csidriver.go b/pkg/operator/ceph/csi/csidriver.go index ec373e872882..19d47408d953 100644 --- a/pkg/operator/ceph/csi/csidriver.go +++ b/pkg/operator/ceph/csi/csidriver.go @@ -28,7 +28,7 @@ import ( ) type csiDriver interface { - createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error + createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string, attachRequired bool) error reCreateCSIDriverInfo(ctx context.Context) error deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error } @@ -39,8 +39,7 @@ type v1CsiDriver struct { } // createCSIDriverInfo Registers CSI driver by creating a CSIDriver object -func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error { - attach := true +func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string, attachRequired bool) error { mountInfo := false // Create CSIDriver object csiDriver := &v1k8scsi.CSIDriver{ @@ -48,7 +47,7 @@ func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernet Name: name, }, Spec: v1k8scsi.CSIDriverSpec{ - AttachRequired: &attach, + AttachRequired: &attachRequired, PodInfoOnMount: &mountInfo, }, } diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 3f052123c891..26163d5c6c04 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -37,6 +37,7 @@ import ( type Param struct { CSIPluginImage string + NFSPluginImage string RegistrarImage string ProvisionerImage string AttacherImage string @@ -47,6 +48,7 @@ type Param struct { KubeletDirPath string ForceCephFSKernelClient string CephFSPluginUpdateStrategy string + NFSPluginUpdateStrategy string RBDPluginUpdateStrategy string PluginPriorityClassName string ProvisionerPriorityClassName string @@ -69,6 +71,7 @@ type Param struct { RBDLivenessMetricsPort uint16 ProvisionerReplicas int32 CSICephFSPodLabels map[string]string + CSINFSPodLabels map[string]string CSIRBDPodLabels map[string]string } @@ -83,12 +86,14 @@ var ( EnableRBD = false EnableCephFS = false + EnableNFS = false EnableCSIGRPCMetrics = false AllowUnsupported = false CustomCSICephConfigExists = false //driver names CephFSDriverName string + NFSDriverName string RBDDriverName string // configuration map for csi @@ -105,6 +110,7 @@ var ( var ( // image names DefaultCSIPluginImage = "quay.io/cephcsi/cephcsi:v3.6.0" + DefaultNFSPluginImage = "mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0" DefaultRegistrarImage = "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0" DefaultProvisionerImage = "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0" DefaultAttacherImage = "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0" @@ -128,6 +134,10 @@ var ( CephFSProvisionerDepTemplatePath string //go:embed template/cephfs/csi-cephfsplugin-svc.yaml CephFSPluginServiceTemplatePath string + + // Local package template path for NFS + //go:embed template/nfs/csi-nfsplugin.yaml + NFSPluginTemplatePath string ) const ( @@ -149,6 +159,10 @@ const ( cephFSPluginTolerationsEnv = "CSI_CEPHFS_PLUGIN_TOLERATIONS" cephFSPluginNodeAffinityEnv = "CSI_CEPHFS_PLUGIN_NODE_AFFINITY" + // NFS tolerations and node affinity + nfsPluginTolerationsEnv = "CSI_NFS_PLUGIN_TOLERATIONS" + nfsPluginNodeAffinityEnv = "CSI_NFS_PLUGIN_NODE_AFFINITY" + // RBD tolerations and node affinity rbdProvisionerTolerationsEnv = "CSI_RBD_PROVISIONER_TOLERATIONS" rbdProvisionerNodeAffinityEnv = "CSI_RBD_PROVISIONER_NODE_AFFINITY" @@ -162,6 +176,8 @@ const ( cephFSProvisionerResource = "CSI_CEPHFS_PROVISIONER_RESOURCE" cephFSPluginResource = "CSI_CEPHFS_PLUGIN_RESOURCE" + nfsPluginResource = "CSI_NFS_PLUGIN_RESOURCE" + // kubelet directory path DefaultKubeletDirPath = "/var/lib/kubelet" @@ -186,14 +202,16 @@ const ( // driver daemonset names csiRBDPlugin = "csi-rbdplugin" csiCephFSPlugin = "csi-cephfsplugin" + csiNFSPlugin = "csi-nfsplugin" // driver deployment names csiRBDProvisioner = "csi-rbdplugin-provisioner" csiCephFSProvisioner = "csi-cephfsplugin-provisioner" + csiNFSProvisioner = "csi-nfsplugin-provisioner" ) func CSIEnabled() bool { - return EnableRBD || EnableCephFS + return EnableRBD || EnableCephFS || EnableNFS } func validateCSIParam() error { @@ -216,7 +234,7 @@ func validateCSIParam() error { func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerInfo, v *CephCSIVersion) error { var ( err error - rbdPlugin, cephfsPlugin *apps.DaemonSet + rbdPlugin, cephfsPlugin, nfsPlugin *apps.DaemonSet rbdProvisionerDeployment, cephfsProvisionerDeployment *apps.Deployment rbdService, cephfsService *corev1.Service ) @@ -230,6 +248,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI CephFSDriverName = tp.DriverNamePrefix + "cephfs.csi.ceph.com" RBDDriverName = tp.DriverNamePrefix + "rbd.csi.ceph.com" + NFSDriverName = tp.DriverNamePrefix + "nfs.csi.ceph.com" csiDriverobj = beta1CsiDriver{} if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver { @@ -331,6 +350,12 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI tp.CephFSPluginUpdateStrategy = rollingUpdate } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + tp.NFSPluginUpdateStrategy = onDelete + } else { + tp.NFSPluginUpdateStrategy = rollingUpdate + } + if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { tp.RBDPluginUpdateStrategy = onDelete } else { @@ -412,6 +437,13 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI cephfsService.Namespace = r.opConfig.OperatorNamespace } + if EnableNFS { + nfsPlugin, err = templateToDaemonSet("nfsplugin", NFSPluginTemplatePath, tp) + if err != nil { + return errors.Wrap(err, "failed to load nfs plugin template") + } + } + // get common provisioner tolerations and node affinity provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) @@ -554,18 +586,50 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } } + if nfsPlugin != nil { + // get NFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified + nfsPluginTolerations := getToleration(r.opConfig.Parameters, nfsPluginTolerationsEnv, pluginTolerations) + nfsPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, nfsPluginNodeAffinityEnv, pluginNodeAffinity) + // apply NFS plugin tolerations and node affinity + applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations) + // apply resource request and limit to nfs plugin containers + applyResourcesToContainers(r.opConfig.Parameters, nfsPluginResource, &nfsPlugin.Spec.Template.Spec) + err = ownerInfo.SetControllerReference(nfsPlugin) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name) + } + multusApplied, err := r.applyCephClusterNetworkConfig(r.opManagerContext, &nfsPlugin.Spec.Template.ObjectMeta) + if err != nil { + return errors.Wrapf(err, "failed to apply network config to nfs plugin daemonset %q", nfsPlugin.Name) + } + if multusApplied { + nfsPlugin.Spec.Template.Spec.HostNetwork = false + } + err = k8sutil.CreateDaemonSet(r.opManagerContext, csiNFSPlugin, r.opConfig.OperatorNamespace, r.context.Clientset, nfsPlugin) + if err != nil { + return errors.Wrapf(err, "failed to start nfs plugin daemonset %q", nfsPlugin.Name) + } + k8sutil.AddRookVersionLabelToDaemonSet(nfsPlugin) + } + if EnableRBD { - err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))) + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)), true) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) } } if EnableCephFS { - err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy))) + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)), true) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) } } + if EnableNFS { + err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, NFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_FSGROUPPOLICY", string(k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy)), false) + if err != nil { + return errors.Wrapf(err, "failed to create CSI driver object for %q", NFSDriverName) + } + } return nil } @@ -573,6 +637,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI func (r *ReconcileCSI) stopDrivers(ver *version.Info) error { RBDDriverName = fmt.Sprintf("%s.rbd.csi.ceph.com", r.opConfig.OperatorNamespace) CephFSDriverName = fmt.Sprintf("%s.cephfs.csi.ceph.com", r.opConfig.OperatorNamespace) + NFSDriverName = fmt.Sprintf("%s.nfs.csi.ceph.com", r.opConfig.OperatorNamespace) if !EnableRBD { logger.Info("CSI Ceph RBD driver disabled") @@ -592,6 +657,15 @@ func (r *ReconcileCSI) stopDrivers(ver *version.Info) error { logger.Info("successfully removed CSI CephFS driver") } + if !EnableNFS { + logger.Info("CSI NFS driver disabled") + err := r.deleteCSIDriverResources(ver, csiNFSPlugin, csiNFSProvisioner, "csi-nfsplugin-metrics", NFSDriverName) + if err != nil { + return errors.Wrap(err, "failed to remove CSI NFS driver") + } + logger.Info("successfully removed CSI NFS driver") + } + return nil } diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml new file mode 100644 index 000000000000..f501a6864e91 --- /dev/null +++ b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-nfsplugin + namespace: {{ .Namespace }} +spec: + selector: + matchLabels: + app: csi-nfsplugin + updateStrategy: + type: {{ .NFSPluginUpdateStrategy }} + template: + metadata: + labels: + app: csi-nfsplugin + {{ range $key, $value := .CSINFSPodLabels }} + {{ $key }}: "{{ $value }}" + {{ end }} + spec: + serviceAccountName: rook-csi-nfs-plugin-sa + hostNetwork: {{ .EnableCSIHostNetwork }} + {{ if .PluginPriorityClassName }} + priorityClassName: {{ .PluginPriorityClassName }} + {{ end }} + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + image: {{ .RegistrarImage }} + args: + - "--v={{ .LogLevel }}" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path={{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}nfs.csi.ceph.com/csi.sock" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-nfsplugin + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + args: + - "--v={{ .LogLevel }}" + - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com" + - --nodeid=$(NODE_ID) + - --endpoint=$(CSI_ENDPOINT) + env: + - name: NODE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + image: {{ .NFSPluginImage }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: "{{ .KubeletDirPath }}/pods" + mountPropagation: "Bidirectional" + volumes: + - name: plugin-dir + hostPath: + path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}nfs.csi.ceph.com/" + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: "{{ .KubeletDirPath }}/plugins_registry/" + type: Directory + - name: pods-mount-dir + hostPath: + path: "{{ .KubeletDirPath }}/pods" + type: Directory