diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index 4a65a84d2a..0921bce98e 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -73,8 +73,10 @@ function prepare_tensorrtllm() { --dtype float16 # 3. Build engine + # max_batch_size set to 128 to avoid OOM errors trtllm-build --checkpoint_dir ${CKPT_PATH} \ --gemm_plugin auto \ + --max_batch_size 128 \ --output_dir ${ENGINE_PATH} # 4. Prepare model repository