Set max batch size to 128

triton-inference-server · Jan 17, 2025 · fd078fd · fd078fd
1 parent 199266e
commit fd078fd
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh
@@ -73,8 +73,10 @@ function prepare_tensorrtllm() {
         --dtype float16
 
     # 3. Build engine
+    # max_batch_size set to 128 to avoid OOM errors
     trtllm-build --checkpoint_dir ${CKPT_PATH} \
         --gemm_plugin auto \
+        --max_batch_size 128 \
         --output_dir ${ENGINE_PATH}
 
     # 4. Prepare model repository