chatml-function-calling

[albertusgeyser]

Good Day

Could anyone guide me how to get "normal" responses from model if no function was detected? or do i understand this incorrectly that i need a chat_format for each "function" i need?

This is my testing code i use:

from llama_cpp import Llama
from datetime import datetime

# Initialize the model with the correct chat format for function calling
model_path = "Mistral-7B-Instruct-v0.3-Q4_K_M.gguf"
llm = Llama(
    model_path=model_path,
    chat_format="chatml-function-calling"
)

# Define the functions to get weather and time information
def get_current_weather(location, unit="celsius"):
    """Get the current weather in a given location"""
    # Mock data for demonstration
    weather_info = {
        "location": location,
        "temperature": 22 if unit == "celsius" else 71.6,
        "unit": unit,
        "forecast": ["sunny", "windy"]
    }
    return json.dumps(weather_info)

def get_time_in_location(location):
    """Get the current time in a given location"""
    # Mock data for demonstration
    current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    time_info = {
        "location": location,
        "current_time": current_time
    }
    return json.dumps(time_info)

# Define the tools (formerly functions) schema
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_current_weather",
            "description": "Get the current weather in a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA"
                    },
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"]
                    }
                },
                "required": ["location"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "get_time_in_location",
            "description": "Get the current time in a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA"
                    }
                },
                "required": ["location"]
            }
        }
    }
]

def get_info(question):
    system_message = """You are a helpful assistant that can call functions to get information when necessary.
    If the user asks about weather or time in a specific location, use the appropriate function.
    For all other questions, respond directly without calling any functions.
    Always provide a complete and coherent response."""

    prompt = f"""Please analyze this question: '{question}'
    If it requires current weather or time information for a specific location, use the appropriate function.
    Otherwise, provide a direct answer without calling any functions."""

    print("Sending request to the model...")
    response = llm.create_chat_completion(
        messages=[
            {"role": "system", "content": system_message},
            {"role": "user", "content": prompt}
        ],
        tools=tools,
        tool_choice="auto",
        response_format={
        "type": "json_object",
    	},
    	temperature=0.7,
    )

    print("\nRaw response from the model:")
    print(json.dumps(response, indent=2))

    if 'choices' in response and len(response['choices']) > 0:
        choice = response['choices'][0]
        
        if 'message' in choice:
            message = choice['message']
            
            if 'tool_calls' in message and message['tool_calls']:
                print("\nFunction calling detected!")
                tool_call = message['tool_calls'][0]
                function_name = tool_call['function']['name']
                function_args = json.loads(tool_call['function']['arguments'])
                
                print(f"Function called: {function_name}")
                print(f"Arguments: {function_args}")

                if function_name == "get_current_weather":
                    return get_current_weather(**function_args)
                elif function_name == "get_time_in_location":
                    return get_time_in_location(**function_args)
            elif 'content' in message and message['content'].strip():
                print("\nNo function calling detected. The model responded directly.")
                return message['content']
            else:
                print("\nUnexpected or empty message content.")
                return "I apologize, but I couldn't generate a proper response. Could you please rephrase your question?"
        else:
            print("\nUnexpected choice format. Using raw choice content.")
            return str(choice)
    else:
        print("\nUnexpected response structure. Check the raw response for more details.")
        return str(response)

def sanitize_output(output):
    if isinstance(output, str):
        # Remove any partial function call strings
        output = output.replace("functions.get_current_weather:", "")
        output = output.replace("functions.get_time_in_location:", "")
        # Trim whitespace
        output = output.strip()
    return output if output else "I apologize, but I couldn't generate a proper response. Could you please rephrase your question?"

# Main loop for user input
if __name__ == "__main__":
    while True:
        question = input("What would you like to know (or type 'exit' to quit)? ")
        if question.lower() == 'exit':
            print("Exiting the program. Goodbye!")
            break

        print(f"\nRequesting information with question: '{question}'...")
        info = get_info(question)
        sanitized_info = sanitize_output(info)
        print("\nFinal information:")
        print(sanitized_info)```

[albertusgeyser]

Anyone able to help me out here?

[xsploit]

this would be dope if it could act like a reg bot with functions