opea-project · yongfengdu · Jan 21, 2025 · Jan 10, 2025 · Ruoyu-y · Jan 17, 2025
@@ -1,32 +1,38 @@
 # data-prep
 
-Helm chart for deploying data-prep microservice.
+Helm chart for deploying OPEA data-prep microservice.
 
-data-prep will use redis and tei service, please specify the endpoints.
+## Installing the chart
 
-## (Option1): Installing the chart separately
+`data-prep` will use TEI for embedding service, and support different vector DB backends.
 
-First, you need to install the tei and redis-vector-db chart, please refer to the [tei](../tei/README.md) and [redis-vector-db](../redis-vector-db/README.md) for more information.
+- TEI: please refer to the [tei](../tei) for more information.
 
-After you've deployted the tei and redis-vector-db chart successfully, please run `kubectl get svc` to get the service endpoint and URL respectively, i.e. `http://tei`, `redis://redis-vector-db:6379`.
+- Redis vector DB: please refer to [redis-vector-db](../redis-vector-db/) for more information.
 
-To install data-prep chart, run the following:
+- Milvus DB: please refer to [milvus-helm](https://github.com/zilliztech/milvus-helm/tree/milvus-4.2.12) for more information.
 
-```console
-cd GenAIInfra/helm-charts/common/data-prep
-export REDIS_URL="redis://redis-vector-db:6379"
-export TEI_EMBEDDING_ENDPOINT="http://tei"
-helm dependency update
-helm install data-prep . --set REDIS_URL=${REDIS_URL} --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT}
-```
+First, you need to install the `tei` helm chart and one of the vector DB service, i.e. `redis-vector-db` chart.
+
+After you've deployed dependency charts successfully, please run `kubectl get svc` to get the service endpoint URL respectively, i.e. `http://tei:80`, `redis://redis-vector-db:6379`.
 
-## (Option2): Installing the chart with dependencies automatically
+To install `data-prep` chart, run the following:
 
 ```console
 cd GenAIInfra/helm-charts/common/data-prep
 helm dependency update
-helm install data-prep . --set redis-vector-db.enabled=true --set tei.enabled=true
+export HFTOKEN="insert-your-huggingface-token-here"
+export TEI_EMBEDDING_ENDPOINT="http://tei"
+
+# Install data-prep with Redis DB backend
+export DATAPREP_BACKEND="REDIS"
+export DB_HOST="redis-vector-db"
+helm install data-prep . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set DATAPREP_BACKEND=${DATAPREP_BACKEND} --set REDIS_HOST=${DB_HOST}
 
+# Install data-prep with Milvus DB backend
+# export DATAPREP_BACKEND="MILVUS"
+# export DB_HOST="milvus"
+# helm install data-prep . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} --set global.HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} --set DATAPREP_BACKEND=${DATAPREP_BACKEND} --set MILVUS_HOST=${DB_HOST}
 ```
 
 ## Verify
@@ -38,21 +44,23 @@ Then run the command `kubectl port-forward svc/data-prep 6007:6007` to expose th
 Open another terminal and run the following command to verify the service if working:
 
 ```console
-curl http://localhost:6007/v1/dataprep  \
+curl http://localhost:6007/v1/dataprep/ingest  \
     -X POST \
     -H "Content-Type: multipart/form-data" \
     -F "files=@./README.md"
 ```
 
 ## Values
 
-| Key                    | Type   | Default                 | Description                              |
-| ---------------------- | ------ | ----------------------- | ---------------------------------------- |
-| image.repository       | string | `"opea/dataprep-redis"` |                                          |
-| service.port           | string | `"6007"`                |                                          |
-| REDIS_URL              | string | `""`                    |                                          |
-| TEI_EMBEDDING_ENDPOINT | string | `""`                    |                                          |
-| global.monitoring      | bool   | `false`                 | See ../../monitoring.md before enabling! |
+| Key                             | Type   | Default   | Description                                                                                             |
+| ------------------------------- | ------ | --------- | ------------------------------------------------------------------------------------------------------- |
+| service.port                    | string | `"6007"`  |                                                                                                         |
+| global.HUGGINGFACEHUB_API_TOKEN | string | `""`      | Your own Hugging Face API token                                                                         |
+| DATAPREP_BACKEND                | string | `"REDIS"` | vector DB backend to use, one of "REDIS", "MILVUS"                                                      |
+| REDIS_HOST                      | string | `""`      | Redis service URL host, only valid for Redis, please see `values.yaml` for other Redis configuration    |
+| MILVUS_HOST                     | string | `""`      | Milvus service URL host, only valid for Milvus, please see `values.yaml` for other Milvus configuration |
+| TEI_EMBEDDING_ENDPOINT          | string | `""`      |                                                                                                         |
+| global.monitoring               | bool   | `false`   | See ../../monitoring.md before enabling!                                                                |
 
 ## Milvus support
 

@@ -3,28 +3,32 @@
 
 milvus:
   enabled: true
+  # Milvus config for standalone mode with no PVC which has minimum requirements for the K8s cluster.
+  # Check https://github.com/zilliztech/milvus-helm/tree/milvus-4.2.12/charts/milvus for more production level configuration.
   cluster:
     enabled: false
   etcd:
     replicaCount: 1
+    persistence:
+      enabled: false
   pulsar:
     enabled: false
   minio:
     mode: standalone
+    persistence:
+      enabled: false
+  standalone:
+    persistence:
+      enabled: false
 redis-vector-db:
   enabled: false
 tei:
   enabled: true
 
-image:
-  repository: opea/dataprep-milvus
-
-port: 6010
+DATAPREP_BACKEND: "MILVUS"
 # text embedding inference service URL, e.g. http://<service-name>:<port>
-#TEI_EMBEDDING_ENDPOINT: "http://embedding-tei:80"
+# TEI_EMBEDDING_ENDPOINT: "http://data-prep-tei:80"
 # milvus DB configurations
-#MILVUS_HOST: "milvustest"
-MILVUS_PORT: "19530"
+# MILVUS_HOST: "data-prep-milvus"
+MILVUS_PORT: 19530
 COLLECTION_NAME: "rag_milvus"
-MOSEC_EMBEDDING_ENDPOINT: ""
-MOSEC_EMBEDDING_MODEL: ""
@@ -8,36 +8,38 @@ metadata:
   labels:
     {{- include "data-prep.labels" . | nindent 4 }}
 data:
-  {{- if .Values.MOSEC_EMBEDDING_ENDPOINT }}
-  MOSEC_EMBEDDING_ENDPOINT: {{ .Values.MOSEC_EMBEDDING_ENDPOINT | quote}}
-  MOSEC_EMBEDDING_MODEL: {{ .Values.MOSEC_EMBEDDING_MODEL | quote}}
-  {{- else if .Values.TEI_EMBEDDING_ENDPOINT }}
-  TEI_ENDPOINT: {{ .Values.TEI_EMBEDDING_ENDPOINT | quote}}
-  TEI_EMBEDDING_ENDPOINT: {{ .Values.TEI_EMBEDDING_ENDPOINT | quote}}
+  {{- if .Values.TEI_EMBEDDING_ENDPOINT }}
+  TEI_EMBEDDING_ENDPOINT: {{ tpl .Values.TEI_EMBEDDING_ENDPOINT . | quote}}
   {{- else if not .Values.LOCAL_EMBEDDING_MODEL }}
-  TEI_ENDPOINT: "http://{{ .Release.Name }}-tei"
+  TEI_EMBEDDING_ENDPOINT: "http://{{ .Release.Name }}-tei"
   {{- end }}
-  {{- if .Values.LOCAL_EMBEDDING_MODEL }}
-  EMBED_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | quote }}
-  LOCAL_EMBEDDING_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | quote }}
-  {{- end }}
-  {{- if .Values.REDIS_URL }}
-  REDIS_URL: {{ .Values.REDIS_URL | quote}}
+  {{- if eq "REDIS" .Values.DATAPREP_BACKEND }}
+  DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_REDIS"
+  {{- if .Values.REDIS_HOST }}
+  REDIS_HOST: {{ tpl .Values.REDIS_HOST . | quote}}
   {{- else }}
-  REDIS_URL: "redis://{{ .Release.Name }}-redis-vector-db:6379"
+  REDIS_HOST: "{{ .Release.Name }}-redis-vector-db"
   {{- end }}
+  REDIS_PORT: {{ .Values.REDIS_PORT | quote }}
+  REDIS_SSL: {{ .Values.REDIS_SSL | quote }}
+  REDIS_PASSWORD: {{ .Values.REDIS_PASSWORD | quote }}
+  REDIS_USERNAME: {{ .Values.REDIS_USERNAME | quote }}
   INDEX_NAME: {{ .Values.INDEX_NAME | quote }}
   KEY_INDEX_NAME: {{ .Values.KEY_INDEX_NAME | quote }}
   SEARCH_BATCH_SIZE: {{ .Values.SEARCH_BATCH_SIZE | quote }}
+  EMBED_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | default "" | quote }}
+  {{- else if eq "MILVUS" .Values.DATAPREP_BACKEND }}
+  DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_MILVUS"
   {{- if .Values.MILVUS_HOST }}
-  MILVUS_HOST: {{ .Values.MILVUS_HOST | quote }}
+  MILVUS_HOST: {{ tpl .Values.MILVUS_HOST . | quote }}
   {{- else }}
   MILVUS_HOST: "{{ .Release.Name }}-milvus"
   {{- end }}
-  MILVUS: {{ .Values.MILVUS_HOST | quote }}
   MILVUS_PORT: {{ .Values.MILVUS_PORT | quote }}
-  {{- if .Values.COLLECTION_NAME }}
   COLLECTION_NAME: {{ .Values.COLLECTION_NAME | quote }}
+  LOCAL_EMBEDDING_MODEL: {{ .Values.LOCAL_EMBEDDING_MODEL | default "" | quote }}
+  {{- else }}
+  {{- cat "Invalid DATAPREP_BACKEND:" .Values.DATAPREP_BACKEND | fail }}
   {{- end }}
   HUGGINGFACEHUB_API_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
   HF_HOME: "/tmp/.cache/huggingface"
@@ -46,8 +48,8 @@ data:
   {{- end }}
   http_proxy: {{ .Values.global.http_proxy | quote }}
   https_proxy: {{ .Values.global.https_proxy | quote }}
-  {{- if and (not .Values.REDIS_URL) (and (not .Values.TEI_EMBEDDING_ENDPOINT) (or .Values.global.http_proxy .Values.global.https_proxy)) }}
-  no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Values.global.no_proxy }}"
+  {{- if and (and (not .Values.MILVUS_HOST ) (not .Values.REDIS_HOST)) (and (not .Values.TEI_EMBEDDING_ENDPOINT) (or .Values.global.http_proxy .Values.global.https_proxy)) }}
+  no_proxy: "{{ .Release.Name }}-tei,{{ .Release.Name }}-redis-vector-db,{{ .Release.Name }}-milvus,{{ .Values.global.no_proxy }}"
   {{- else }}
   no_proxy: {{ .Values.global.no_proxy | quote }}
   {{- end }}

@@ -28,8 +28,38 @@ spec:
       serviceAccountName: {{ include "data-prep.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: wait-for-db
+          envFrom:
+            - configMapRef:
+                name: {{ include "data-prep.fullname" . }}-config
+            {{- if .Values.global.extraEnvConfig }}
+            - configMapRef:
+                name: {{ .Values.global.extraEnvConfig }}
+                optional: true
+            {{- end }}
+          image: busybox:1.36
+          command: ["sh", "-c"]
+          args:
+            - |
+              {{- if eq "REDIS" .Values.DATAPREP_BACKEND }}
+              TESTHOST=$(REDIS_HOST);
+              TESTPORT=$(REDIS_PORT);
+              {{- else if eq "MILVUS" .Values.DATAPREP_BACKEND }}
+              TESTHOST=$(MILVUS_HOST);
+              TESTPORT=$(MILVUS_PORT);
+              {{- end }}
+              wait_timeout={{ .Values.waitTimeout | default 1 }};
+              total_timeout={{ .Values.retryTimeoutSeconds | default 60 }};
+              j=0;
+              while ! nc -w ${wait_timeout} -z ${TESTHOST} ${TESTPORT}; do
+                j=$((j+wait_timeout));
+                [[ $j -ge ${total_timeout} ]] && echo "ERROR: ${TESTHOST}:${TESTPORT} is NOT reachable in $j seconds!" && exit 1;
+                j=$((j+2)); sleep 2;
+              done;
+              echo "${TESTHOST}:${TESTPORT} is reachable within $j seconds.";
       containers:
-        - name: {{ .Release.Name }}
+        - name: {{ .Chart.Name }}
           envFrom:
             - configMapRef:
                 name: {{ include "data-prep.fullname" . }}-config

@@ -19,17 +19,24 @@ spec:
         - |
           echo "test file" > /tmp/file1.txt;
           max_retry=20;
+          echo "test upload...";
           for ((i=1; i<=max_retry; i++)); do
-            curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep -sS --fail-with-body \
+            curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep/ingest -sS --fail-with-body \
             -X POST \
             -H "Content-Type: multipart/form-data" \
             -F "files=@/tmp/file1.txt" && break;
             curlcode=$?
             if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
           done;
-          curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep/delete_file -sS \
-          -X POST \
-          -H "Content-Type: application/json" \
-          -d '{"file_path": "file1.txt"}';
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
+          echo "test delete...";
+          for ((i=1; i<=max_retry; i++)); do
+              curl http://{{ include "data-prep.fullname" . }}:{{ .Values.service.port }}/v1/dataprep/delete -sS --fail-with-body \
+              -X POST \
+              -H "Content-Type: application/json" \
+              -d '{"file_path": "all"}' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
           if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
@@ -5,17 +5,46 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-tei:
-  enabled: false
-milvus:
-  enabled: false
-redis-vector-db:
-  enabled: false
+# Configurations for OPEA microservice data-prep
+# Set it as a non-null string, such as true, if you want to enable logging.
+LOGFLAG: ""
+
+# data-prep needs to talk to different vector-DB backend, i.e. Redis, Milvus
+# Default is to use Redis as vector-DB backend
+DATAPREP_BACKEND: "REDIS"
+REDIS_HOST: ""
+REDIS_PORT: 6379
+REDIS_SSL: false
+REDIS_PASSWORD: ""
+REDIS_USERNAME: ""
+INDEX_NAME: "rag_redis"
+KEY_INDEX_NAME: "file-keys"
+SEARCH_BATCH_SIZE: 10
+
+# Uncomment and set the following settings to use Milvus as vector-DB backend
+# DATAPREP_BACKEND: "MILVUS"
+MILVUS_HOST: ""
+MILVUS_PORT: 19530
+COLLECTION_NAME: "rag_milvus"
+
+# data-prep can do embedding locally or talk to remote TEI service to do embedding.
+# TEI_EMBEDDING_ENDPOINT takes precedence over LOCAL_EMBEDDING_MODEL.
+# TEI inference service URL, e.g. http://<service-name>:<port>
+TEI_EMBEDDING_ENDPOINT: ""
+# local embedding model
+LOCAL_EMBEDDING_MODEL: ""
+
+# The following 2 settings are used to wait for dependency DB service to be ready.
+# Normally we don't need to set them unless the init-containers keeps failing.
+# Timeout in seconds for each wait
+# waitTimeout: 1
+# Total timeout in seconds for all wait
+# retryTimeoutSeconds: 60
 
 replicaCount: 1
 
 image:
-  repository: opea/dataprep-redis
+  repository: opea/dataprep
   # Uncomment the following line to set desired image pull policy if needed, as one of Always, IfNotPresent, Never.
   # pullPolicy: ""
   # Overrides the image tag whose default is the chart appVersion.
@@ -52,30 +81,19 @@ securityContext:
   seccompProfile:
     type: RuntimeDefault
 
-port: 6007
+port: 5000
 service:
   type: ClusterIP
   port: 6007
 
 resources:
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
   # limits:
   #   cpu: 100m
   #   memory: 128Mi
   requests:
     cpu: 100m
     memory: 128Mi
 
-livenessProbe:
-  httpGet:
-    path: v1/health_check
-    port: data-prep
-  initialDelaySeconds: 5
-  periodSeconds: 5
-  failureThreshold: 24
 readinessProbe:
   httpGet:
     path: v1/health_check
@@ -96,29 +114,6 @@ tolerations: []
 
 affinity: {}
 
-# Set it as a non-null string, such as true, if you want to enable logging facility,
-# otherwise, keep it as "" to disable it.
-LOGFLAG: ""
-
-# text embedding inference service URL, e.g. http://<service-name>:<port>
-TEI_EMBEDDING_ENDPOINT: ""
-
-# local embedder's model
-LOCAL_EMBEDDING_MODEL: ""
-
-# redis DB service URL, e.g. redis://<service-name>:<port>
-REDIS_URL: ""
-INDEX_NAME: "rag-redis"
-KEY_INDEX_NAME: "file-keys"
-SEARCH_BATCH_SIZE: 10
-
-# milvus DB configurations
-MILVUS_HOST: ""
-MILVUS_PORT: ""
-COLLECTION_NAME: ""
-MOSEC_EMBEDDING_ENDPOINT: ""
-MOSEC_EMBEDDING_MODEL: ""
-
 global:
   http_proxy: ""
   https_proxy: ""
@@ -134,3 +129,11 @@ global:
 
   # Prometheus Helm install release name needed for serviceMonitors
   prometheusRelease: prometheus-stack
+
+# For CI tests only
+tei:
+  enabled: false
+milvus:
+  enabled: false
+redis-vector-db:
+  enabled: false