Trying to using functions with VLLM

[joestein-ssc]

I followed the directions here https://github.com/vllm-project/vllm/blob/main/docs/source/serving/openai_compatible_server.md#tool-calling-in-the-chat-completion-api-1 but now I am getting this error. I am using the Llama3.1-70b-instruct model.

My code.

model = "meta-llama/Meta-Llama-3.1-70B-Instruct"

# Define the tools (functions) you want the model to use
def get_current_weather(location: str) -> str:
    # Pretend we're querying a weather API
    return f"The weather in {location} is sunny and 25°C."

def calculate_factorial(n: int) -> int:
    # Simple factorial calculation
    if n == 0 or n == 1:
        return 1
    return n * calculate_factorial(n - 1)

# Define the functions in OpenAI format for function calling
functions = [
    {
        "name": "get_current_weather",
        "description": "Get the current weather in a given location.",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {
                    "type": "string",
                    "description": "The location for which to get the weather.",
                },
            },
            "required": ["location"],
        },
    },
    {
        "name": "calculate_factorial",
        "description": "Calculate the factorial of a number.",
        "parameters": {
            "type": "object",
            "properties": {
                "n": {
                    "type": "integer",
                    "description": "A positive integer to calculate its factorial.",
                },
            },
            "required": ["n"],
        },
    },
]

client = OpenAI(
    api_key=ssnc_api_key,
    base_url=url,
)

# Query the model and let it decide when to call a function
def ask_openai(query):
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": "You are a helpful assistant"},
            {"role": "user", "content": query},
        ],
        functions=functions,
    )

    # Check if a function was called
    message = response["choices"][0]["message"]
    if "function_call" in message:
        function_name = message["function_call"]["name"]
        arguments = json.loads(message["function_call"]["arguments"])

        # Handle the function call
        if function_name == "get_current_weather":
            result = get_current_weather(arguments["location"])
        elif function_name == "calculate_factorial":
            result = calculate_factorial(arguments["n"])

        # Send the result of the function back to the model for a full response
        follow_up_response = client.chat.completions.create(
            model=model,
            messages=[
                {"role": "system", "content": "You are a helpful assistant"},
                {"role": "user", "content": query},
                message,  # The original function call
                {"role": "function", "name": function_name, "content": str(result)},
            ]
        )
        return follow_up_response["choices"][0]["message"]["content"]
    else:
        # If no function was called, return the original response
        return message["content"]

# Example queries
query1 = "What is the weather like in New York?"
query2 = "What is the factorial of 5?"

print("Response 1:", ask_openai(query1))
print("Response 2:", ask_openai(query2))

The error

Traceback (most recent call last):
  File "/Users/jstein/dev/ssnc/ai/tarvos/test_tools.py", line 105, in <module>
    print("Response 1:", ask_openai(query1))
                         ^^^^^^^^^^^^^^^^^^
  File "/Users/jstein/dev/ssnc/ai/tarvos/test_tools.py", line 65, in ask_openai
    response = client.chat.completions.create(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/openai/_utils/_utils.py", line 274, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/openai/resources/chat/completions.py", line 668, in create
    return self._post(
           ^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/openai/_base_client.py", line 1260, in post
    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/openai/_base_client.py", line 937, in request
    return self._request(
           ^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/openai/_base_client.py", line 1041, in _request
    raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'object': 'error', 'message': "[{'type': 'extra_forbidden', 'loc': ('body', 'functions'), 'msg': 'Extra inputs are not permitted', 'input': [{'name': 'get_current_weather', 'description': 'Get the current weather in a given location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location for which to get the weather.'}}, 'required': ['location']}}, {'name': 'calculate_factorial', 'description': 'Calculate the factorial of a number.', 'parameters': {'type': 'object', 'properties': {'n': {'type': 'integer', 'description': 'A positive integer to calculate its factorial.'}}, 'required': ['n']}}]}]", 'type': 'BadRequestError', 'param': None, 'code': 400}