Support multiple card for LLM inference

DeepSeek models require more than one Gaudi cards. Defined NUM_CARDS for user to configure. Pass NUM_CARDS to container as compose does. Pass NUM_SHARD to tgi and --tensor-parellel-size for vllm. Fix CI detect helm chart issue. Signed-off-by: Dolpher Du <[email protected]>
opea-project · Feb 12, 2025 · 26efce2 · 26efce2
1 parent 3016f5f
commit 26efce2
Show file tree

Hide file tree

Showing 14 changed files with 32 additions and 14 deletions.
diff --git a/.github/workflows/push-release-charts.yaml b/.github/workflows/push-release-charts.yaml
@@ -50,12 +50,13 @@ jobs:
           # Update Examples
           e2e_charts=$(git diff --name-only ${base_commit} ${merged_commit} | \
           grep "^$CHARTS_DIR" | \
-          grep -vE 'README.md|valuefiles.yaml|common|*.sh' | \
+          grep -vE 'valuefiles.yaml|common|*.md|*.sh' | \
           cut -d'/' -f2 | sort -u )
           echo "Charts to be updated: $e2e_charts"
           echo "${{ secrets.ACTION_TOKEN }}" | helm registry login ghcr.io -u opea --password-stdin
           pushd $CHARTS_DIR
           for chart in ${e2e_charts}; do
+            if [ -f $chart ]; then continue; fi
             echo "Updating $chart"
             helm dependency update ${chart}
             helm package $chart

diff --git a/helm-charts/agentqna/values.yaml b/helm-charts/agentqna/values.yaml
@@ -136,7 +136,8 @@ tgi:
   LIMIT_HPU_GRAPH: true
   USE_FLASH_ATTENTION: true
   FLASH_ATTENTION_RECOMPUTE: true
-  extraCmdArgs: ["--sharded", "true", "--num-shard", "4"]
+  NUM_CARDS: 4
+  SHARDED: "true"
 
 vllm:
   enabled: false
@@ -151,7 +152,8 @@ vllm:
   OMPI_MCA_btl_vader_single_copy_mechanism: none
   PT_HPU_ENABLE_LAZY_COLLECTIVES: true
   VLLM_SKIP_WARMUP: true
-  extraCmdArgs: ["--tensor-parallel-size", "4", "--max-seq_len-to-capture", "16384"]
+  NUM_CARDS: 4
+  extraCmdArgs: ["--max-seq_len-to-capture", "16384"]
 
 nginx:
   service:

diff --git a/helm-charts/chatqna/gaudi-vllm-values.yaml b/helm-charts/chatqna/gaudi-vllm-values.yaml
@@ -31,9 +31,8 @@ vllm:
 
   PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
   OMPI_MCA_btl_vader_single_copy_mechanism: "none"
-
+  NUM_CARDS: 1
   extraCmdArgs: [
-    "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
     "--max-seq_len-to-capture", "2048"

diff --git a/helm-charts/chatqna/guardrails-gaudi-values.yaml b/helm-charts/chatqna/guardrails-gaudi-values.yaml
@@ -103,9 +103,8 @@ vllm:
 
   PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
   OMPI_MCA_btl_vader_single_copy_mechanism: "none"
-
+  NUM_CARDS: 1
   extraCmdArgs: [
-    "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
     "--max-seq_len-to-capture", "2048"

diff --git a/helm-charts/common/agent/values.yaml b/helm-charts/common/agent/values.yaml
@@ -24,7 +24,7 @@ tgi:
   LIMIT_HPU_GRAPH: true
   USE_FLASH_ATTENTION: true
   FLASH_ATTENTION_RECOMPUTE: true
-  extraCmdArgs: ["--sharded", "true", "--num-shard", "4"]
+  NUM_CARDS: 4
 
 vllm:
   enabled: false
@@ -38,7 +38,8 @@ vllm:
   OMPI_MCA_btl_vader_single_copy_mechanism: none
   PT_HPU_ENABLE_LAZY_COLLECTIVES: true
   VLLM_SKIP_WARMUP: true
-  extraCmdArgs: ["--tensor-parallel-size", "4", "--max-seq_len-to-capture", "16384"]
+  NUM_CARDS: 4
+  extraCmdArgs: ["--max-seq_len-to-capture", "16384"]
 
 replicaCount: 1
 

diff --git a/helm-charts/common/llm-uservice/vllm-docsum-gaudi-values.yaml b/helm-charts/common/llm-uservice/vllm-docsum-gaudi-values.yaml
@@ -20,7 +20,8 @@ vllm:
     tag: "latest"
   LLM_MODEL_ID: Intel/neural-chat-7b-v3-3
   OMPI_MCA_btl_vader_single_copy_mechanism: none
-  extraCmdArgs: ["--tensor-parallel-size","1","--block-size","128","--max-num-seqs","256","--max-seq_len-to-capture","2048"]
+  NUM_CARDS: 1
+  extraCmdArgs: ["--block-size", "128", "--max-num-seqs", "256", "--max-seq_len-to-capture", "2048"]
   resources:
     limits:
       habana.ai/gaudi: 1
diff --git a/helm-charts/common/llm-uservice/vllm-gaudi-values.yaml b/helm-charts/common/llm-uservice/vllm-gaudi-values.yaml
@@ -13,7 +13,8 @@ vllm:
     tag: "latest"
   LLM_MODEL_ID: Intel/neural-chat-7b-v3-3
   OMPI_MCA_btl_vader_single_copy_mechanism: none
-  extraCmdArgs: ["--tensor-parallel-size","1","--block-size","128","--max-num-seqs","256","--max-seq_len-to-capture","2048"]
+  NUM_CARDS: 1
+  extraCmdArgs: ["--block-size", "128", "--max-num-seqs", "256", "--max-seq_len-to-capture", "2048"]
   resources:
     limits:
       habana.ai/gaudi: 1

diff --git a/helm-charts/common/tgi/templates/configmap.yaml b/helm-charts/common/tgi/templates/configmap.yaml
@@ -64,3 +64,10 @@ data:
   {{- if .Values.BATCH_BUCKET_SIZE }}
   BATCH_BUCKET_SIZE: {{ .Values.BATCH_BUCKET_SIZE | quote }}
   {{- end }}
+  {{- if .Values.SHARDED }}
+  SHARDED: {{ .Values.SHARDED }}
+  {{- end }}
+  {{- if .Values.NUM_CARDS }}
+  NUM_CARDS: {{ .Values.NUM_CARDS | quote }}
+  NUM_SHARD: {{ .Values.NUM_CARDS | quote }}
+  {{- end }}
diff --git a/helm-charts/common/tgi/values.yaml b/helm-charts/common/tgi/values.yaml
@@ -133,6 +133,7 @@ MAX_TOTAL_TOKENS: ""
 CUDA_GRAPHS: "0"
 HF_HUB_DISABLE_PROGRESS_BARS: "1"
 HF_HUB_ENABLE_HF_TRANSFER: "0"
+NUM_CARDS: "1"
 
 global:
   http_proxy: ""

diff --git a/helm-charts/common/vllm/gaudi-values.yaml b/helm-charts/common/vllm/gaudi-values.yaml
@@ -8,7 +8,8 @@ image:
 
 # VLLM_CPU_KVCACHE_SPACE: "40"
 OMPI_MCA_btl_vader_single_copy_mechanism: none
-extraCmdArgs: ["--tensor-parallel-size","1","--block-size","128","--max-num-seqs","256","--max-seq_len-to-capture","2048"]
+extraCmdArgs: ["--block-size","128","--max-num-seqs","256","--max-seq_len-to-capture","2048"]
+NUM_CARDS: "1"
 resources:
   limits:
     habana.ai/gaudi: 1
diff --git a/helm-charts/common/vllm/templates/configmap.yaml b/helm-charts/common/vllm/templates/configmap.yaml
@@ -37,3 +37,6 @@ data:
   {{- if .Values.VLLM_TORCH_PROFILER_DIR }}
   VLLM_TORCH_PROFILER_DIR: {{ .Values.VLLM_TORCH_PROFILER_DIR | quote }}
   {{- end }}
+  {{- if .Values.NUM_CARDS }}
+  NUM_CARDS: {{ .Values.NUM_CARDS | quote }}
+  {{- end }}
diff --git a/helm-charts/common/vllm/templates/deployment.yaml b/helm-charts/common/vllm/templates/deployment.yaml
@@ -90,6 +90,8 @@ spec:
             - {{ . | quote }}
             {{- end }}
           {{- end }}
+            - "--tensor-parallel-size"
+            - {{ .Values.NUM_CARDS | quote }}
             - "--model"
             - {{ .Values.LLM_MODEL_ID | quote }}
             - "--host"

diff --git a/helm-charts/common/vllm/values.yaml b/helm-charts/common/vllm/values.yaml
@@ -109,6 +109,7 @@ LLM_MODEL_ID: Intel/neural-chat-7b-v3-3
 OMPI_MCA_btl_vader_single_copy_mechanism: ""
 PT_HPU_ENABLE_LAZY_COLLECTIVES: ""
 VLLM_CPU_KVCACHE_SPACE: ""
+NUM_CARDS: "1"
 
 global:
   http_proxy: ""

diff --git a/helm-charts/docsum/gaudi-vllm-values.yaml b/helm-charts/docsum/gaudi-vllm-values.yaml
@@ -34,9 +34,8 @@ vllm:
 
   PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
   OMPI_MCA_btl_vader_single_copy_mechanism: "none"
-
+  NUM_CARDS: 1
   extraCmdArgs: [
-    "--tensor-parallel-size", "1",
     "--block-size", "128",
     "--max-num-seqs", "256",
     "--max-seq_len-to-capture", "2048"