From eae843029573d79a5a879aa77643916da45abecf Mon Sep 17 00:00:00 2001 From: Izzy Putterman Date: Tue, 21 May 2024 15:41:46 -0700 Subject: [PATCH] Patch vLLM for missing content entry (#671) * Patch vLLM * Formatting --- src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py index 3b6194a68..6b1b9e2bd 100755 --- a/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py +++ b/src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py @@ -620,6 +620,11 @@ def _preprocess_response( responses = response.strip().split("\n\n") if len(responses) > 1: merged_response = json.loads(remove_sse_prefix(responses[0])) + if ( + merged_response["choices"][0]["delta"].get("content", None) + is None + ): + merged_response["choices"][0]["delta"]["content"] = "" for r in responses[1:]: text = self._extract_openai_text_output(r) merged_response["choices"][0]["delta"]["content"] += text