From 3a019d0657fd6c730212fdff85899ab9cb9305e6 Mon Sep 17 00:00:00 2001 From: Ji Bin Date: Thu, 17 Jun 2021 22:18:59 +0800 Subject: [PATCH] feature: add s3 storage support (#180) As milvus 1.1.1 has been released, and s3 is supported default by official images. This supports s3 storage for cluster deployment with helm. related issue: https://github.com/milvus-io/milvus/issues/5335 Signed-off-by: Ji Bin --- charts/milvus/Chart.yaml | 2 +- charts/milvus/README.md | 27 +++++++++++++++++++ .../templates/_readonly_server_config.tpl | 8 ++++++ charts/milvus/templates/_server_config.tpl | 8 ++++++ charts/milvus/templates/pvc.yaml | 2 ++ .../milvus/templates/readonly-deployment.yaml | 9 +++++++ .../milvus/templates/writable-deployment.yaml | 9 +++++++ charts/milvus/values.yaml | 7 +++++ 8 files changed, 71 insertions(+), 1 deletion(-) diff --git a/charts/milvus/Chart.yaml b/charts/milvus/Chart.yaml index ff5baf9..339313a 100755 --- a/charts/milvus/Chart.yaml +++ b/charts/milvus/Chart.yaml @@ -3,7 +3,7 @@ name: milvus appVersion: "1.1.1" kubeVersion: "^1.10.0-0" description: Milvus is an open source similarity search engine for massive-scalefeature vectors. -version: 1.1.4 +version: 1.1.5 keywords: - milvus - elastic diff --git a/charts/milvus/README.md b/charts/milvus/README.md index dff3d2b..254d780 100644 --- a/charts/milvus/README.md +++ b/charts/milvus/README.md @@ -58,6 +58,27 @@ PVC are shared properly between your pods: To share a PV with multiple Pods, the PV needs to have accessMode 'ReadOnlyMany' or 'ReadWriteMany'. +#### Deploying cluster with s3 storage support +```bash +$ helm install --set cluster.enabled=true \ + --set persistence.enabled=false \ + --set storage.s3.enabled=true \ + --set storage.s3.address=ip_or_hostname \ + --set storage.s3.port=port_number \ + --set storage.s3.access_key=access_key \ + --set storage.s3.secret_key=secret_key \ + --set storage.s3.bucket=bucket \ + --set mysql.persistence.storageClass=storage_class_for_mysql \ + my-release . +``` + +Notes: +- When s3 storage is enabled, persistent segment data will using s3, you don't need to setup shared PC/PVC for + readonly/writable instances. +- For mysql database instance, you should still prepare PV/PVC(ReadWriteOnce) for it. +- Currently by Milvus's 1.1.1 implementation, only with http is supported. https based s3 currently not supported. + + ## Uninstall the Chart To uninstall/delete the my-release deployment: @@ -95,6 +116,12 @@ The following table lists the configurable parameters of the Milvus chart and th | `cache.insertBufferSize` | Maximum insert buffer size allowed (GB) | `1GB` | | `cache.cacheSize` | Size of CPU memory used for cache (GB) | `4GB` | | `network.httpPort` | Port that Milvus web server monitors. | `19121` | +| `storage.s3.enabled` | Enable s3 storage as persistent storage. | `false` | +| `storage.s3.address` | S3 service's address, IP or hostname. | `unset` | +| `storage.s3.port` | S3 service's port. | `unset` | +| `storage.s3.access_key` | S3 service's access key. | `unset` | +| `storage.s3.secret_key` | S3 service's secret key. | `unset` | +| `storage.s3.bucket | S3 service's bucket name. | `unset` | | `wal.enabled` | Enable write-ahead logging. | `true` | | `wal.recoveryErrorIgnore` | Whether to ignore logs with errors that happens during WAL | `true` | | `wal.bufferSize` | Sum total of the read buffer and the write buffer. (MB) | `256MB` | diff --git a/charts/milvus/templates/_readonly_server_config.tpl b/charts/milvus/templates/_readonly_server_config.tpl index c8ad40e..2d5433d 100644 --- a/charts/milvus/templates/_readonly_server_config.tpl +++ b/charts/milvus/templates/_readonly_server_config.tpl @@ -72,6 +72,14 @@ network: storage: path: {{ .Values.primaryPath }} auto_flush_interval: {{ .Values.storage.autoFlushInterval }} + {{- if .Values.storage.s3.enabled }} + s3_enabled: true + s3_address: {{ .Values.storage.s3.address }} + s3_port: {{ .Values.storage.s3.port }} + s3_access_key: {{ .Values.storage.s3.access_key }} + s3_secret_key: {{ .Values.storage.s3.secret_key }} + s3_bucket: {{ .Values.storage.s3.bucket }} + {{- end }} #----------------------+------------------------------------------------------------+------------+-----------------+ # WAL Config | Description | Type | Default | diff --git a/charts/milvus/templates/_server_config.tpl b/charts/milvus/templates/_server_config.tpl index 32581ed..6641c20 100644 --- a/charts/milvus/templates/_server_config.tpl +++ b/charts/milvus/templates/_server_config.tpl @@ -72,6 +72,14 @@ network: storage: path: {{ .Values.primaryPath }} auto_flush_interval: {{ .Values.storage.autoFlushInterval }} + {{- if .Values.storage.s3.enabled }} + s3_enabled: true + s3_address: {{ .Values.storage.s3.address }} + s3_port: {{ .Values.storage.s3.port }} + s3_access_key: {{ .Values.storage.s3.access_key }} + s3_secret_key: {{ .Values.storage.s3.secret_key }} + s3_bucket: {{ .Values.storage.s3.bucket }} + {{- end }} #----------------------+------------------------------------------------------------+------------+-----------------+ # WAL Config | Description | Type | Default | diff --git a/charts/milvus/templates/pvc.yaml b/charts/milvus/templates/pvc.yaml index b498f71..60a20cb 100644 --- a/charts/milvus/templates/pvc.yaml +++ b/charts/milvus/templates/pvc.yaml @@ -1,5 +1,6 @@ {{- $pvc := .Values.persistence.persistentVolumeClaim -}} {{- if and .Values.persistence.enabled (not $pvc.existingClaim) }} +{{- if not .Values.storage.s3.enabled }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -24,6 +25,7 @@ spec: requests: storage: {{ $pvc.size }} {{- end }} +{{- end }} {{- $pvc := .Values.logsPersistence.persistentVolumeClaim -}} {{- if and .Values.logsPersistence.enabled (not $pvc.existingClaim) }} diff --git a/charts/milvus/templates/readonly-deployment.yaml b/charts/milvus/templates/readonly-deployment.yaml index 03c2106..5c490d0 100755 --- a/charts/milvus/templates/readonly-deployment.yaml +++ b/charts/milvus/templates/readonly-deployment.yaml @@ -46,15 +46,22 @@ spec: command: - 'sh' - '-c' + {{- if .Values.storage.s3.enabled }} + - > + echo "TODO: introduce s3 client and put a text file to s3 storage" + {{- else }} - > until test -f {{ .Values.primaryPath }}/db/test/{{ template "milvus.fullname" . }}.txt && echo read from share storage; do sleep 2; done; + {{- end }} volumeMounts: + {{- if not .Values.storage.s3.enabled }} - name: milvus-data-disk mountPath: {{ .Values.persistence.mountPath | quote }} subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }} readOnly: true + {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -98,9 +105,11 @@ spec: {{ toYaml .Values.extraEnv | indent 8 }} {{- end }} volumeMounts: + {{- if not .Values.storage.s3.enabled }} - name: milvus-data-disk mountPath: {{ .Values.persistence.mountPath | quote }} subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }} + {{- end }} - name: milvus-logs-disk mountPath: {{ .Values.logsPersistence.mountPath | quote }} subPath: {{ .Values.logsPersistence.persistentVolumeClaim.subPath | default "" }} diff --git a/charts/milvus/templates/writable-deployment.yaml b/charts/milvus/templates/writable-deployment.yaml index c056c68..19b4a12 100755 --- a/charts/milvus/templates/writable-deployment.yaml +++ b/charts/milvus/templates/writable-deployment.yaml @@ -50,15 +50,22 @@ spec: command: - 'sh' - '-c' + {{- if .Values.storage.s3.enabled }} + - > + echo "TODO: introduce s3 client and put a text file to s3 storage" + {{- else }} - > mkdir -p {{ .Values.primaryPath }}/db/test; until touch {{ .Values.primaryPath }}/db/test/{{ template "milvus.fullname" . }}.txt && echo write to share storage; do sleep 2; done; + {{- end }} volumeMounts: + {{- if not .Values.storage.s3.enabled }} - name: milvus-data-disk mountPath: {{ .Values.persistence.mountPath | quote }} subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }} + {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -103,9 +110,11 @@ spec: {{ toYaml .Values.extraEnv | indent 8 }} {{- end }} volumeMounts: + {{- if not .Values.storage.s3.enabled }} - name: milvus-data-disk mountPath: {{ .Values.persistence.mountPath | quote }} subPath: {{ .Values.persistence.persistentVolumeClaim.subPath | default "" }} + {{- end }} - name: milvus-logs-disk mountPath: {{ .Values.logsPersistence.mountPath | quote }} subPath: {{ .Values.logsPersistence.persistentVolumeClaim.subPath | default "" }} diff --git a/charts/milvus/values.yaml b/charts/milvus/values.yaml index 2d22928..85c0088 100755 --- a/charts/milvus/values.yaml +++ b/charts/milvus/values.yaml @@ -23,6 +23,13 @@ network: storage: autoFlushInterval: 1 + s3: + enabled: false + address: 127.0.0.1 + port: 9000 + access_key: replace_with_your_access_key + secret_key: replace_with_your_secret_key + bucket: milvus wal: enabled: true