feature: add s3 storage support (#180)

As milvus 1.1.1 has been released, and s3 is supported default by official images. This supports s3 storage for cluster deployment with helm. related issue: milvus-io/milvus#5335 Signed-off-by: Ji Bin <[email protected]>
zilliztech · Jun 17, 2021 · 3a019d0 · 3a019d0
1 parent 8995dcf
commit 3a019d0
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 1 deletion.
diff --git a/charts/milvus/Chart.yaml b/charts/milvus/Chart.yaml
@@ -3,7 +3,7 @@ name: milvus
 appVersion: "1.1.1"
 kubeVersion: "^1.10.0-0"
 description: Milvus is an open source similarity search engine for massive-scalefeature vectors.
-version: 1.1.4
+version: 1.1.5
 keywords:
   - milvus
   - elastic

diff --git a/charts/milvus/README.md b/charts/milvus/README.md
@@ -58,6 +58,27 @@ PVC are shared properly between your pods:
 
 To share a PV with multiple Pods, the PV needs to have accessMode 'ReadOnlyMany' or 'ReadWriteMany'.
 
+#### Deploying cluster with s3 storage support
+```bash
+$ helm install --set cluster.enabled=true \
+               --set persistence.enabled=false \
+               --set storage.s3.enabled=true \
+               --set storage.s3.address=ip_or_hostname \
+               --set storage.s3.port=port_number \
+               --set storage.s3.access_key=access_key \
+               --set storage.s3.secret_key=secret_key \
+               --set storage.s3.bucket=bucket \
+               --set mysql.persistence.storageClass=storage_class_for_mysql \
+               my-release .
+```
+
+Notes:
+- When s3 storage is enabled, persistent segment data will using s3, you don't need to setup shared PC/PVC for
+  readonly/writable instances.
+- For mysql database instance, you should still prepare PV/PVC(ReadWriteOnce) for it.
+- Currently by Milvus's 1.1.1 implementation, only with http is supported. https based s3 currently not supported.
+
+
 ## Uninstall the Chart
 
 To uninstall/delete the my-release deployment:
@@ -95,6 +116,12 @@ The following table lists the configurable parameters of the Milvus chart and th
 | `cache.insertBufferSize`                  | Maximum insert buffer size allowed (GB)       | `1GB`                                                   |
 | `cache.cacheSize`                         | Size of CPU memory used for cache  (GB)       | `4GB`                                                   |
 | `network.httpPort`                        | Port that Milvus web server monitors.         | `19121`                                                 |
+| `storage.s3.enabled`                      | Enable s3 storage as persistent storage.      | `false`                                                 |
+| `storage.s3.address`                      | S3 service's address, IP or hostname.         | `unset`                                                 |
+| `storage.s3.port`                         | S3 service's port.                            | `unset`                                                 |
+| `storage.s3.access_key`                   | S3 service's access key.                      | `unset`                                                 |
+| `storage.s3.secret_key`                   | S3 service's secret key.                      | `unset`                                                 |
+| `storage.s3.bucket                        | S3 service's bucket name.                     | `unset`                                                 |
 | `wal.enabled`                             | Enable write-ahead logging.                   | `true`                                                  |
 | `wal.recoveryErrorIgnore`                 | Whether to ignore logs with errors that happens during WAL | `true`                                     |
 | `wal.bufferSize`                          | Sum total of the read buffer and the write buffer. (MB) | `256MB`                                       |

diff --git a/charts/milvus/templates/_readonly_server_config.tpl b/charts/milvus/templates/_readonly_server_config.tpl
@@ -72,6 +72,14 @@ network:
 storage:
   path: {{ .Values.primaryPath }}
   auto_flush_interval: {{ .Values.storage.autoFlushInterval }}
+  {{- if .Values.storage.s3.enabled }}
+  s3_enabled: true
+  s3_address: {{ .Values.storage.s3.address }}
+  s3_port: {{ .Values.storage.s3.port }}
+  s3_access_key: {{ .Values.storage.s3.access_key }}
+  s3_secret_key: {{ .Values.storage.s3.secret_key }}
+  s3_bucket: {{ .Values.storage.s3.bucket }}
+  {{- end }}
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # WAL Config           | Description                                                | Type       | Default         |

diff --git a/charts/milvus/templates/_server_config.tpl b/charts/milvus/templates/_server_config.tpl
@@ -72,6 +72,14 @@ network:
 storage:
   path: {{ .Values.primaryPath }}
   auto_flush_interval: {{ .Values.storage.autoFlushInterval }}
+  {{- if .Values.storage.s3.enabled }}
+  s3_enabled: true
+  s3_address: {{ .Values.storage.s3.address }}
+  s3_port: {{ .Values.storage.s3.port }}
+  s3_access_key: {{ .Values.storage.s3.access_key }}
+  s3_secret_key: {{ .Values.storage.s3.secret_key }}
+  s3_bucket: {{ .Values.storage.s3.bucket }}
+  {{- end }}
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # WAL Config           | Description                                                | Type       | Default         |

diff --git a/charts/milvus/templates/pvc.yaml b/charts/milvus/templates/pvc.yaml
@@ -1,5 +1,6 @@
 {{- $pvc := .Values.persistence.persistentVolumeClaim -}}
 {{- if and .Values.persistence.enabled (not $pvc.existingClaim) }}
+{{- if not .Values.storage.s3.enabled }}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -24,6 +25,7 @@ spec:
     requests:
       storage: {{ $pvc.size }}
 {{- end }}
+{{- end }}
 
 {{- $pvc := .Values.logsPersistence.persistentVolumeClaim -}}
 {{- if and .Values.logsPersistence.enabled (not $pvc.existingClaim) }}

diff --git a/charts/milvus/templates/readonly-deployment.yaml b/charts/milvus/templates/readonly-deployment.yaml
@@ -46,15 +46,22 @@ spec:
         command:
         - 'sh'
         - '-c'
+        {{- if .Values.storage.s3.enabled }}
+        - >
+          echo "TODO: introduce s3 client and put a text file to s3 storage"
+        {{- else }}
         - >
           until test -f {{ .Values.primaryPath }}/db/test/{{ template "milvus.fullname" . }}.txt && echo read from share storage; do
             sleep 2;
           done;
+        {{- end }}
         volumeMounts:
+        {{- if not .Values.storage.s3.enabled }}
         - name: milvus-data-disk
           mountPath: {{ .Values.persistence.mountPath | quote }}
           subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }}
           readOnly: true
+        {{- end }}
         {{- with .Values.extraVolumeMounts }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -98,9 +105,11 @@ spec:
 {{ toYaml .Values.extraEnv | indent 8 }}
         {{- end }}
         volumeMounts:
+        {{- if not .Values.storage.s3.enabled }}
         - name: milvus-data-disk
           mountPath: {{ .Values.persistence.mountPath | quote }}
           subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }}
+        {{- end }}
         - name: milvus-logs-disk
           mountPath: {{ .Values.logsPersistence.mountPath | quote }}
           subPath: {{ .Values.logsPersistence.persistentVolumeClaim.subPath | default "" }}

diff --git a/charts/milvus/templates/writable-deployment.yaml b/charts/milvus/templates/writable-deployment.yaml
@@ -50,15 +50,22 @@ spec:
         command:
         - 'sh'
         - '-c'
+        {{- if .Values.storage.s3.enabled }}
+        - >
+          echo "TODO: introduce s3 client and put a text file to s3 storage"
+        {{- else }}
         - >
           mkdir -p {{ .Values.primaryPath }}/db/test;
           until touch {{ .Values.primaryPath }}/db/test/{{ template "milvus.fullname" . }}.txt && echo write to share storage; do
             sleep 2;
           done;
+        {{- end }}
         volumeMounts:
+        {{- if not .Values.storage.s3.enabled }}
         - name: milvus-data-disk
           mountPath: {{ .Values.persistence.mountPath | quote }}
           subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }}
+        {{- end }}
         {{- with .Values.extraVolumeMounts }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -103,9 +110,11 @@ spec:
 {{ toYaml .Values.extraEnv | indent 8 }}
         {{- end }}
         volumeMounts:
+        {{- if not .Values.storage.s3.enabled }}
         - name: milvus-data-disk
           mountPath: {{ .Values.persistence.mountPath | quote }}
           subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }}
+        {{- end }}
         - name: milvus-logs-disk
           mountPath: {{ .Values.logsPersistence.mountPath | quote }}
           subPath: {{ .Values.logsPersistence.persistentVolumeClaim.subPath | default "" }}

diff --git a/charts/milvus/values.yaml b/charts/milvus/values.yaml
@@ -23,6 +23,13 @@ network:
 
 storage:
   autoFlushInterval: 1
+  s3:
+    enabled: false
+    address: 127.0.0.1
+    port: 9000
+    access_key: replace_with_your_access_key
+    secret_key: replace_with_your_secret_key
+    bucket: milvus
 
 wal:
   enabled: true