Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[serve][doc] update serve vllm openai example for latest vllm version #50192

Merged
merged 9 commits into from
Feb 6, 2025
28 changes: 20 additions & 8 deletions doc/source/serve/doc_code/vllm_openai_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@
ErrorResponse,
)
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.entrypoints.openai.serving_engine import LoRAModulePath, PromptAdapterPath
from vllm.entrypoints.openai.serving_models import (
BaseModelPath,
LoRAModulePath,
PromptAdapterPath,
OpenAIServingModels,
)

from vllm.utils import FlexibleArgumentParser
from vllm.entrypoints.logger import RequestLogger

Expand Down Expand Up @@ -67,19 +73,25 @@ async def create_chat_completion(
if not self.openai_serving_chat:
model_config = await self.engine.get_model_config()
# Determine the name of the served model for the OpenAI client.
if self.engine_args.served_model_name is not None:
served_model_names = self.engine_args.served_model_name
else:
served_model_names = [self.engine_args.model]
self.openai_serving_chat = OpenAIServingChat(
models = OpenAIServingModels(
self.engine,
model_config,
served_model_names,
self.response_role,
[
BaseModelPath(
name=self.engine_args.model, model_path=self.engine_args.model
)
],
lora_modules=self.lora_modules,
prompt_adapters=self.prompt_adapters,
)
self.openai_serving_chat = OpenAIServingChat(
self.engine,
model_config,
models,
self.response_role,
request_logger=self.request_logger,
chat_template=self.chat_template,
chat_template_content_format="auto",
)
logger.info(f"Request: {request}")
generator = await self.openai_serving_chat.create_chat_completion(
Expand Down