Patch vLLM for missing content entry (#671)

* Patch vLLM * Formatting
triton-inference-server · May 21, 2024 · eae8430 · eae8430
1 parent bba524d
commit eae8430
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py
@@ -620,6 +620,11 @@ def _preprocess_response(
                 responses = response.strip().split("\n\n")
                 if len(responses) > 1:
                     merged_response = json.loads(remove_sse_prefix(responses[0]))
+                    if (
+                        merged_response["choices"][0]["delta"].get("content", None)
+                        is None
+                    ):
+                        merged_response["choices"][0]["delta"]["content"] = ""
                     for r in responses[1:]:
                         text = self._extract_openai_text_output(r)
                         merged_response["choices"][0]["delta"]["content"] += text