diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 42fd430aad92f..d48057e0f123e 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -2,9 +2,9 @@ # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py import time from argparse import Namespace -from typing import Any, Dict, List, Literal, Optional, Union from random import choices from string import ascii_letters, digits +from typing import Any, Dict, List, Literal, Optional, Sequence, Union import torch from openai.types.chat import ChatCompletionContentPartParam @@ -805,7 +805,7 @@ class ExtractedToolCallInformation(BaseModel): tools_called: bool # extracted tool calls - tool_calls: List[ToolCall | MistralToolCall] + tool_calls: Sequence[Union[ToolCall, MistralToolCall]] # content - per OpenAI spec, content AND tool calls can be returned rarely # But some models will do this intentionally @@ -815,7 +815,8 @@ class ExtractedToolCallInformation(BaseModel): class ChatMessage(OpenAIBaseModel): role: str content: Optional[str] = None - tool_calls: List[ToolCall | MistralToolCall] = Field(default_factory=list) + tool_calls: Sequence[Union[ToolCall, MistralToolCall]] = Field( + default_factory=list) class ChatCompletionLogProb(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index df953c5d2d78f..4219324b2cab2 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -71,7 +71,7 @@ def extract_tool_calls(self, # load the JSON, and then use it to build the Function and # Tool Call function_call_arr = json.loads(raw_tool_call) - tool_calls: List[MistralToolCall] = [ + tool_calls: Sequence[MistralToolCall] = [ MistralToolCall( type="function", function=FunctionCall(