[Frontend] OpenAI Responses API supports Tool/Function calling

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Author: chaunceyjiang

vllm/entrypoints/openai/serving_chat.py

@@ -6,7 +6,7 @@
 import time
 from collections.abc import AsyncGenerator, AsyncIterator
 from collections.abc import Sequence as GenericSequence
-from typing import Callable, Final, Optional, Union
+from typing import Final, Optional, Union
 
 import jinja2
 import partial_json_parser
@@ -31,13 +31,12 @@
 from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                     clamp_prompt_logprobs)
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
-from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
+from vllm.entrypoints.openai.tool_parsers import ToolParser
 from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
     MistralToolCall)
 from vllm.entrypoints.utils import get_max_tokens
 from vllm.logger import init_logger
 from vllm.outputs import CompletionOutput, RequestOutput
-from vllm.reasoning import ReasoningParser, ReasoningParserManager
 from vllm.sampling_params import BeamSearchParams, SamplingParams
 from vllm.sequence import Logprob
 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
@@ -78,39 +77,16 @@ def __init__(
         self.chat_template = chat_template
         self.chat_template_content_format: Final = chat_template_content_format
 
-        # set up tool use
         self.enable_auto_tools: bool = enable_auto_tools
-        if self.enable_auto_tools:
-            logger.info(
-                "\"auto\" tool choice has been enabled please note that while"
-                " the parallel_tool_calls client option is preset for "
-                "compatibility reasons, it will be ignored.")
-
-        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
-                                                 ReasoningParser]] = None
-        if reasoning_parser:
-            try:
-                self.reasoning_parser = (
-                    ReasoningParserManager.get_reasoning_parser(
-                        reasoning_parser))
-                assert self.reasoning_parser is not None
-            except Exception as e:
-                raise TypeError(
-                    f"{reasoning_parser=} has not been registered") from e
-        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
-        if self.enable_auto_tools:
-            try:
-                if (tool_parser == "pythonic" and
-                        model_config.model.startswith("meta-llama/Llama-3.2")):
-                    logger.warning(
-                        "Llama3.2 models may struggle to emit valid pythonic"
-                        " tool calls")
-                self.tool_parser = ToolParserManager.get_tool_parser(
-                    tool_parser)
-            except Exception as e:
-                raise TypeError("Error: --enable-auto-tool-choice requires "
-                                f"tool_parser:'{tool_parser}' which has not "
-                                "been registered") from e
+        # set up reasoning parser
+        self.reasoning_parser = self._get_reasoning_parser(
+            reasoning_parser_name=reasoning_parser)
+        # set up tool use
+        self.tool_parser = self._get_tool_parser(
+            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools)
+
+        self.expand_tools_even_if_tool_choice_none = (
+            expand_tools_even_if_tool_choice_none)
 
         self.enable_prompt_tokens_details = enable_prompt_tokens_details
         self.enable_force_include_usage = enable_force_include_usage

vllm/entrypoints/openai/serving_engine.py

@@ -57,7 +57,7 @@
                                               TranscriptionResponse,
                                               TranslationRequest)
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
-from vllm.entrypoints.openai.tool_parsers import ToolParser
+from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
 # yapf: enable
 from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt
 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
@@ -69,6 +69,7 @@
 from vllm.outputs import PoolingRequestOutput, RequestOutput
 from vllm.pooling_params import PoolingParams
 from vllm.prompt_adapter.request import PromptAdapterRequest
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
 from vllm.sampling_params import BeamSearchParams, SamplingParams
 from vllm.sequence import Logprob, PromptLogprobs
 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -1047,6 +1048,51 @@ def _get_model_name(self,
             return self.models.base_model_paths[0].name
         return model_name
 
+    def _get_tool_parser(
+        self,
+        tool_parser_name: str,
+        enable_auto_tools: bool = False
+    ) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
+        """Get the tool parser based on the name."""
+        parser = None
+        if not enable_auto_tools:
+            return parser
+        logger.info(
+            "\"auto\" tool choice has been enabled please note that while"
+            " the parallel_tool_calls client option is preset for "
+            "compatibility reasons, it will be ignored.")
+        """Get the tool parser based on the name."""
+        try:
+            if (tool_parser_name == "pythonic"
+                    and self.model_config.model.startswith(
+                        "meta-llama/Llama-3.2")):
+                logger.warning(
+                    "Llama3.2 models may struggle to emit valid pythonic"
+                    " tool calls")
+            parser = ToolParserManager.get_tool_parser(tool_parser_name)
+        except Exception as e:
+            raise TypeError("Error: --enable-auto-tool-choice requires "
+                            f"tool_parser:'{tool_parser_name}' which has not "
+                            "been registered") from e
+        return parser
+
+    def _get_reasoning_parser(
+        self,
+        reasoning_parser_name: str,
+    ) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
+        """Get the reasoning parser based on the name."""
+        parser = None
+        if not reasoning_parser_name:
+            return None
+        try:
+            parser = (ReasoningParserManager.get_reasoning_parser(
+                reasoning_parser_name))
+            assert parser is not None
+        except Exception as e:
+            raise TypeError(
+                f"{reasoning_parser_name=} has not been registered") from e
+        return parser
+
 
 def clamp_prompt_logprobs(
     prompt_logprobs: Union[PromptLogprobs,

vllm/entrypoints/openai/serving_responses.py

@@ -6,10 +6,15 @@
 import time
 from collections.abc import AsyncGenerator, AsyncIterator
 from http import HTTPStatus
-from typing import Callable, Final, Optional, Union
+from typing import Final, Optional, Union
 
 import jinja2
 from fastapi import Request
+from openai.types.chat import (ChatCompletionAssistantMessageParam,
+                               ChatCompletionMessageToolCallParam,
+                               ChatCompletionToolMessageParam)
+from openai.types.chat.chat_completion_message_tool_call_param import (
+    Function as FunctionCallTool)
 from openai.types.responses import (ResponseFunctionToolCall,
                                     ResponseOutputMessage, ResponseOutputText,
                                     ToolChoiceFunction)
@@ -32,10 +37,8 @@
 # yapf: enable
 from vllm.entrypoints.openai.serving_engine import OpenAIServing
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
-from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
 from vllm.logger import init_logger
 from vllm.outputs import RequestOutput
-from vllm.reasoning import ReasoningParser, ReasoningParserManager
 from vllm.sampling_params import SamplingParams
 from vllm.transformers_utils.tokenizer import AnyTokenizer
 from vllm.utils import random_fc_uuid, random_uuid
@@ -72,30 +75,14 @@ def __init__(
         self.enable_auto_tools = enable_auto_tools
         self.expand_tools_even_if_tool_choice_none = (
             expand_tools_even_if_tool_choice_none)
-        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
-        if self.enable_auto_tools:
-            try:
-                self.tool_parser = ToolParserManager.get_tool_parser(
-                    tool_parser)
-            except Exception as e:
-                raise TypeError("Error: --enable-auto-tool-choice requires "
-                                f"tool_parser:'{tool_parser}' which has not "
-                                "been registered") from e
         self.chat_template = chat_template
         self.chat_template_content_format: Final = chat_template_content_format
 
-        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
-                                                 ReasoningParser]] = None
-        if reasoning_parser:
-            try:
-                self.reasoning_parser = (
-                    ReasoningParserManager.get_reasoning_parser(
-                        reasoning_parser))
-                assert self.reasoning_parser is not None
-            except Exception as e:
-                raise TypeError(
-                    f"{reasoning_parser=} has not been registered") from e
+        self.reasoning_parser = self._get_reasoning_parser(
+            reasoning_parser_name=reasoning_parser)
 
+        self.tool_parser = self._get_tool_parser(
+            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools)
         self.enable_prompt_tokens_details = enable_prompt_tokens_details
         self.enable_force_include_usage = enable_force_include_usage
         self.default_sampling_params = (
@@ -480,25 +467,28 @@ def _construct_input_messages(
             for item in request.input:
                 if item.get("type") == "function_call":
                     # Append the function call as a tool call.
-                    messages.append({
-                        "role":
-                        "assistant",
-                        "tool_calls": [{
-                            "id": item.get("call_id"),
-                            "function": {
-                                "name": item.get("name"),
-                                "arguments": item.get("arguments", "{}"),
-                            },
-                            "type": "function",
-                        }]
-                    })
+                    messages.append(
+                        ChatCompletionAssistantMessageParam(
+                            role="assistant",
+                            tool_calls=[
+                                ChatCompletionMessageToolCallParam(
+                                    id=item.get("call_id"),
+                                    function=FunctionCallTool(
+                                        name=item.get("name"),
+                                        arguments=item.get("arguments", "{}"),
+                                    ),
+                                    type="function",
+                                )
+                            ],
+                        ))
                 elif item.get("type") == "function_call_output":
                     # Append the function call output as a tool message.
-                    messages.append({
-                        "role": "tool",
-                        "content": item.get("output", ""),
-                        "tool_call_id": item.get("call_id"),
-                    })
+                    messages.append(
+                        ChatCompletionToolMessageParam(
+                            role="tool",
+                            content=item.get("output", ""),
+                            tool_call_id=item.get("call_id"),
+                        ))
                 else:
                     messages.append(item)  # type: ignore
         return messages