From b4abcdef7dc674b4342513483ba4f112663a1828 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Thu, 28 Dec 2023 18:31:10 +0000 Subject: [PATCH 1/4] Making helm charts configurable --- .../templates/node.yaml | 28 ++++++++++++++----- .../aws-mountpoint-s3-csi-driver/values.yaml | 11 +++++++- deploy/kubernetes/base/node-daemonset.yaml | 20 +++++++++++++ pkg/driver/mount.go | 2 +- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml index e04d08d1..0e499a91 100644 --- a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml +++ b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml @@ -61,11 +61,13 @@ spec: value: /host/dev/ptmx # mount-s3 runs in systemd context, so this is relative to the host - name: MOUNT_S3_PATH - value: /opt/mountpoint-s3-csi/bin/mount-s3 + value: {{ default "/opt/mountpoint-s3-csi/bin/" .Values.node.mountpointInstallPath }}mount-s3 - name: CSI_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + - name: HOST_TOKEN_PATH + value: {{ printf "%s/plugins/%s" .Values.node.kubeletPath (default "s3.csi.aws.com/token" .Values.node.hostTokenPath) }} {{- with .Values.awsAccessSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: @@ -82,7 +84,7 @@ spec: {{- end }} volumeMounts: - name: kubelet-dir - mountPath: /var/lib/kubelet + mountPath: {{ .Values.node.kubeletPath }} - name: plugin-dir mountPath: /csi - name: systemd-bus @@ -103,6 +105,10 @@ spec: timeoutSeconds: 3 periodSeconds: 2 failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} - name: node-driver-registrar image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} @@ -113,7 +119,7 @@ spec: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/s3.csi.aws.com/csi.sock + value: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -132,6 +138,10 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} @@ -140,6 +150,10 @@ spec: volumeMounts: - name: plugin-dir mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: host-dev hostPath: @@ -147,7 +161,7 @@ spec: type: Directory - name: mp-install hostPath: - path: /opt/mountpoint-s3-csi/bin/ + path: {{ default "/opt/mountpoint-s3-csi/bin/" .Values.node.mountpointInstallPath }} type: DirectoryOrCreate - name: proc-mounts hostPath: @@ -159,15 +173,15 @@ spec: type: Socket - name: kubelet-dir hostPath: - path: /var/lib/kubelet + path: {{ .Values.node.kubeletPath }} type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/s3.csi.aws.com/ + path: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/ type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: {{ .Values.node.kubeletPath }}/plugins_registry/ type: Directory {{- with .Values.node.volumes }} {{- toYaml . | nindent 8 }} diff --git a/charts/aws-mountpoint-s3-csi-driver/values.yaml b/charts/aws-mountpoint-s3-csi-driver/values.yaml index c527760f..46d74baf 100644 --- a/charts/aws-mountpoint-s3-csi-driver/values.yaml +++ b/charts/aws-mountpoint-s3-csi-driver/values.yaml @@ -11,6 +11,8 @@ image: node: resources: {} kubeletPath: /var/lib/kubelet + mountpointInstallPath: /opt/mountpoint-s3-csi/bin/ # should end with "/" + hostTokenPath: s3.csi.aws.com/token # the end path will be `kubeletPath + "/plugins/" + hostTokenpath` logLevel: 4 containerSecurityContext: privileged: true @@ -18,7 +20,12 @@ node: # Specifies whether a service account should be created create: true name: s3-csi-driver-sa - + resources: + requests: + cpu: 10m + memory: 40Mi + limits: + memory: 256Mi sidecars: nodeDriverRegistrar: image: @@ -35,6 +42,7 @@ sidecars: mountPath: /csi - name: registration-dir mountPath: /registration + resources: {} livenessProbe: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe @@ -43,6 +51,7 @@ sidecars: volumeMounts: - mountPath: /csi name: plugin-dir + resources: {} nameOverride: "" fullnameOverride: "" diff --git a/deploy/kubernetes/base/node-daemonset.yaml b/deploy/kubernetes/base/node-daemonset.yaml index 414bf6fd..a1101ce0 100644 --- a/deploy/kubernetes/base/node-daemonset.yaml +++ b/deploy/kubernetes/base/node-daemonset.yaml @@ -72,6 +72,8 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: HOST_TOKEN_PATH + value: /var/lib/kubelet/plugins/s3.csi.aws.com/token volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -95,6 +97,12 @@ spec: initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 2 + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi - name: node-driver-registrar image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.7.0-eks-1-23-13 imagePullPolicy: IfNotPresent @@ -115,6 +123,12 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi - name: liveness-probe image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.9.0-eks-1-23-13 imagePullPolicy: IfNotPresent @@ -124,6 +138,12 @@ spec: volumeMounts: - mountPath: /csi name: plugin-dir + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi volumes: - name: host-dev hostPath: diff --git a/pkg/driver/mount.go b/pkg/driver/mount.go index 0c118369..6867f091 100644 --- a/pkg/driver/mount.go +++ b/pkg/driver/mount.go @@ -39,11 +39,11 @@ const ( defaultRegionEnv = "AWS_DEFAULT_REGION" stsEndpointsEnv = "AWS_STS_REGIONAL_ENDPOINTS" MountS3PathEnv = "MOUNT_S3_PATH" + hostTokenPath = "HOST_TOKEN_PATH" defaultMountS3Path = "/usr/bin/mount-s3" procMounts = "/host/proc/mounts" userAgentPrefix = "--user-agent-prefix" csiDriverPrefix = "s3-csi-driver/" - hostTokenPath = "/var/lib/kubelet/plugins/s3.csi.aws.com/token" ) // Mounter is an interface for mount operations From 467933dcbfbe455ad49db2639f4135c0f3e78206 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Thu, 28 Dec 2023 21:01:46 +0000 Subject: [PATCH 2/4] Read the token path from env in the driver --- pkg/driver/mount.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/driver/mount.go b/pkg/driver/mount.go index 6867f091..b3e8f769 100644 --- a/pkg/driver/mount.go +++ b/pkg/driver/mount.go @@ -39,7 +39,7 @@ const ( defaultRegionEnv = "AWS_DEFAULT_REGION" stsEndpointsEnv = "AWS_STS_REGIONAL_ENDPOINTS" MountS3PathEnv = "MOUNT_S3_PATH" - hostTokenPath = "HOST_TOKEN_PATH" + hostTokenPathEnv = "HOST_TOKEN_PATH" defaultMountS3Path = "/usr/bin/mount-s3" procMounts = "/host/proc/mounts" userAgentPrefix = "--user-agent-prefix" @@ -195,6 +195,11 @@ func passthroughEnv() []string { } webIdentityFile := os.Getenv(webIdentityTokenEnv) awsRoleArn := os.Getenv(roleArnEnv) + hostTokenPath := os.Getenv(hostTokenPathEnv) + if hostTokenPath == "" { + // set the default in case the env variable isn't found + hostTokenPath = "/var/lib/kubelet/plugins/s3.csi.aws.com/token" + } if webIdentityFile != "" { env = append(env, webIdentityTokenEnv+"="+hostTokenPath) env = append(env, roleArnEnv+"="+awsRoleArn) From f09e9ad8712294471d9d9290eafb3b180c08c311 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Tue, 2 Jan 2024 20:27:32 +0000 Subject: [PATCH 3/4] Only make the kubelet path configurable --- charts/aws-mountpoint-s3-csi-driver/templates/node.yaml | 2 +- charts/aws-mountpoint-s3-csi-driver/values.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml index 0e499a91..eef2285c 100644 --- a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml +++ b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml @@ -67,7 +67,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: HOST_TOKEN_PATH - value: {{ printf "%s/plugins/%s" .Values.node.kubeletPath (default "s3.csi.aws.com/token" .Values.node.hostTokenPath) }} + value: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/token {{- with .Values.awsAccessSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: diff --git a/charts/aws-mountpoint-s3-csi-driver/values.yaml b/charts/aws-mountpoint-s3-csi-driver/values.yaml index 46d74baf..f7d89b78 100644 --- a/charts/aws-mountpoint-s3-csi-driver/values.yaml +++ b/charts/aws-mountpoint-s3-csi-driver/values.yaml @@ -12,7 +12,6 @@ node: resources: {} kubeletPath: /var/lib/kubelet mountpointInstallPath: /opt/mountpoint-s3-csi/bin/ # should end with "/" - hostTokenPath: s3.csi.aws.com/token # the end path will be `kubeletPath + "/plugins/" + hostTokenpath` logLevel: 4 containerSecurityContext: privileged: true From 61370d92910492e38af486ceebb931e3f11620e4 Mon Sep 17 00:00:00 2001 From: Dev Lakhia Date: Wed, 3 Jan 2024 16:13:43 +0000 Subject: [PATCH 4/4] Trim suffix / --- charts/aws-mountpoint-s3-csi-driver/templates/node.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml index eef2285c..e8acdece 100644 --- a/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml +++ b/charts/aws-mountpoint-s3-csi-driver/templates/node.yaml @@ -67,7 +67,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: HOST_TOKEN_PATH - value: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/token + value: {{ trimSuffix "/" .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/token {{- with .Values.awsAccessSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: @@ -119,7 +119,7 @@ spec: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/csi.sock + value: {{ trimSuffix "/" .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -177,11 +177,11 @@ spec: type: Directory - name: plugin-dir hostPath: - path: {{ .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/ + path: {{ trimSuffix "/" .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/ type: DirectoryOrCreate - name: registration-dir hostPath: - path: {{ .Values.node.kubeletPath }}/plugins_registry/ + path: {{ trimSuffix "/" .Values.node.kubeletPath }}/plugins_registry/ type: Directory {{- with .Values.node.volumes }} {{- toYaml . | nindent 8 }}