support pulsar v3, update helm & kubernetes' minimum version

.github/workflows/lint-test.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Set up Helm
         uses: azure/setup-helm@v1
         with:
-          version: v3.4.0
+          version: v3.12.3
 
       - name: Add dependency chart repos
         run: |

charts/milvus/Chart.yaml

@@ -3,7 +3,7 @@ name: milvus
 appVersion: "2.4.16"
 kubeVersion: "^1.10.0-0"
 description: Milvus is an open-source vector database built to power AI applications and vector similarity search.
-version: 4.2.20
+version: 4.2.21
 keywords:
   - milvus
   - elastic
@@ -18,4 +18,3 @@ maintainers:
   - email: [email protected]
     name: contact
     url: milvus.io
-engine: gotpl

charts/milvus/README.md

@@ -9,8 +9,8 @@ This chart bootstraps Milvus deployment on a Kubernetes cluster using the Helm p
 
 ## Prerequisites
 
-- Kubernetes 1.14+ (Attu requires 1.18+)
-- Helm >= 3.2.0
+- Kubernetes 1.20+
+- Helm >= 3.12.3
 
 ## Compatibility Notice
 As of version 4.2.0, the Milvus Helm chart no longer supports Milvus v2.3.x. If you need to deploy Milvus v2.3.x using Helm, please use Milvus Helm chart version less than 4.2.0 (e.g 4.1.36).
@@ -566,6 +566,12 @@ The following table lists the configurable parameters of the Milvus Mixture Coor
 ### Pulsar Configuration
 
 This version of the chart includes the dependent Pulsar chart in the charts/ directory.
+- `pulsar-v3.3.0` is used for Pulsar v3
+- `pulsar-v2.7.8` is used for Pulsar v2
+
+Since milvus chart version 4.2.21, pulsar v3 is supported, but pulsar v2 will be still used by default until the release of Milvus v2.5.0. 
+
+We recommend creating new instances with pulsar v3 to avoid security vulnerabilities & some bugs in pulsar v2. To use pulsar v3, set `pulsarv3.enabled` to `true` and `pulsar.enabled` to `false`. Set other values for pulsar v3 under `pulsarv3` field.
 
 You can find more information at:
 * [https://pulsar.apache.org/charts](https://pulsar.apache.org/charts)

charts/milvus/ci/cluster-values.yaml

@@ -7,88 +7,62 @@ minio:
   mode: standalone
   resources:
     requests:
-      memory: 1024Mi
+      memory: 100Mi
+  persistence:
+    size: 40Gi
 pulsar:
+  enabled: false
+pulsarv3:
   enabled: true
-  components:
-    pulsar_manager: false
   autorecovery:
     resources:
       requests:
-        cpu: 0.1
-        memory: 256Mi
+        cpu: 0.01
+        memory: 50Mi
   proxy:
     replicaCount: 1
+    autoscaling:
+      enabled: false
     resources:
       requests:
-        cpu: 0.2
-        memory: 256Mi
+        memory: 64Mi
+        cpu: 0.001
     configData:
       PULSAR_MEM: >
-        -Xms256m -Xmx256m
-      PULSAR_GC: >
-         -XX:MaxDirectMemorySize=256m
+        -Xms64m -Xmx512m -XX:MaxDirectMemorySize=2048m
   bookkeeper:
-    replicaCount: 2
+    component: bookie
+    replicaCount: 3
     resources:
       requests:
-        cpu: 0.2
-        memory: 512Mi
+        memory: 64Mi
+        cpu: 0.001
+    volumes:
+      persistence: true
+      journal:
+        size: 20Gi
+      ledgers:
+        size: 40Gi
     configData:
       PULSAR_MEM: >
-        -Xms512m -Xmx512m
-      PULSAR_GC: >
-        -XX:MaxDirectMemorySize=512m
-        -Dio.netty.leakDetectionLevel=disabled
-        -Dio.netty.recycler.linkCapacity=1024
-        -XX:+UseG1GC -XX:MaxGCPauseMillis=10
-        -XX:+ParallelRefProcEnabled
-        -XX:+UnlockExperimentalVMOptions
-        -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32
-        -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50
-        -XX:+DisableExplicitGC
-        -XX:-ResizePLAB
-        -XX:+ExitOnOutOfMemoryError
-        -XX:+PerfDisableSharedMem
-        -XX:+PrintGCDetails
+        -Xms64m
+        -Xmx4096m
+        -XX:MaxDirectMemorySize=8192m
   zookeeper:
     replicaCount: 1
     resources:
       requests:
-        cpu: 0.1
-        memory: 256Mi
+        memory: 64Mi
+        cpu: 0.001
     configData:
       PULSAR_MEM: >
-        -Xms256m
-        -Xmx256m
-      PULSAR_GC: >
-        -Dcom.sun.management.jmxremote
-        -Djute.maxbuffer=10485760
-        -XX:+ParallelRefProcEnabled
-        -XX:+UnlockExperimentalVMOptions
-        -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC
-        -XX:+PerfDisableSharedMem
-        -Dzookeeper.forceSync=no
+        -Xms64m -Xmx256m
   broker:
     replicaCount: 1
     resources:
       requests:
-        cpu: 0.2
-        memory: 512Mi
+        memory: 64Mi
+        cpu: 0.001
     configData:
       PULSAR_MEM: >
-        -Xms512m
-        -Xmx512m
-      PULSAR_GC: >
-        -XX:MaxDirectMemorySize=512m
-        -Dio.netty.leakDetectionLevel=disabled
-        -Dio.netty.recycler.linkCapacity=1024
-        -XX:+ParallelRefProcEnabled
-        -XX:+UnlockExperimentalVMOptions
-        -XX:+DoEscapeAnalysis
-        -XX:ParallelGCThreads=32
-        -XX:ConcGCThreads=32
-        -XX:G1NewSizePercent=50
-        -XX:+DisableExplicitGC
-        -XX:-ResizePLAB
-        -XX:+ExitOnOutOfMemoryError
+        -Xms64m -Xmx4096m -XX:MaxDirectMemorySize=8192m

charts/milvus/requirements.lock

@@ -8,8 +8,11 @@ dependencies:
 - name: pulsar
   repository: https://pulsar.apache.org/charts
   version: 2.7.8
+- name: pulsar
+  repository: https://pulsar.apache.org/charts
+  version: 3.3.0
 - name: kafka
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
   version: 15.5.1
-digest: sha256:74a42f201ec912067249ae71c08e1ed6cf14f5ec21b350801340b1286816f9a8
-generated: "2024-05-06T15:58:35.219386+08:00"
+digest: sha256:6f855ae95c4f79e728ceb308eec8e60a5e609c3f8f5aceadd4f98778569c67ab
+generated: "2024-11-19T16:13:41.631526+08:00"

charts/milvus/requirements.yaml

@@ -17,6 +17,13 @@ dependencies:
   condition: pulsar.enabled
   tags:
     - pulsar
+- name: pulsar
+  alias: pulsarv3
+  version: 3.3.0
+  repository: https://pulsar.apache.org/charts
+  condition: pulsarv3.enabled
+  tags:
+    - pulsarv3
 - name: kafka
   version: 15.5.1
   repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami

charts/milvus/templates/config.tpl

@@ -110,6 +110,22 @@ pulsar:
 {{- end }}
   port: {{ .Values.pulsar.proxy.ports.pulsar }}
   maxMessageSize: {{ .Values.pulsar.maxMessageSize }}
+
+{{- else if .Values.pulsarv3.enabled }}
+
+mq:
+  type: pulsar
+
+messageQueue: pulsar
+
+pulsar:
+{{- if contains .Values.pulsarv3.name .Release.Name }}
+  address: {{ .Release.Name }}-proxy
+{{- else }}
+  address: {{ .Release.Name }}-{{ .Values.pulsarv3.name }}-proxy
+{{- end }}
+  port: {{ .Values.pulsarv3.proxy.ports.pulsar }}
+  maxMessageSize: {{ .Values.pulsarv3.broker.configData.maxMessageSize }}
 {{- end }}
 
 {{- if .Values.externalKafka.enabled }}