Tracing UI in Azure AI Foundry Not Displaying User Inputs, Outputs, or Tool Executions Body

[spasegirl]

I'm developing an agent in Azure AI Foundry that utilizes a custom function tool. The agent operates as expected, and I receive the anticipated outputs. However, in the Azure AI Foundry Tracing UI, I cannot see:

User inputs

Assistant outputs

Tool execution details (the "Tools" section appears empty)

`

from typing import Any, Callable, Set

import os, time, json
from azure.ai.projects import AIProjectClient
from azure.ai.projects.telemetry import trace_function
from azure.identity import DefaultAzureCredential
from azure.ai.projects.models import FunctionTool, RequiredFunctionToolCall, SubmitToolOutputsAction, ToolOutput
from opentelemetry import trace
from azure.monitor.opentelemetry import configure_azure_monitor

project_client = AIProjectClient.from_connection_string(
    credential=DefaultAzureCredential(), conn_str=os.environ["AIPROJECT_CONNECTION_STRING"]
)



application_insights_connection_string = project_client.telemetry.get_connection_string()
if not application_insights_connection_string:
    print("Application Insights was not enabled for this project.")
    print("Enable it via the 'Tracing' tab in your AI Foundry project page.")
    exit()
configure_azure_monitor(connection_string=application_insights_connection_string)


project_client.telemetry.enable()


tracer = trace.get_tracer(__name__)

import sys
project_client.telemetry.enable(destination=sys.stdout)

functions = FunctionTool(functions=user_functions)
toolset = ToolSet()
toolset.add(functions)


functions = FunctionTool(functions=user_functions)

with tracer.start_as_current_span("scenario2"):
    with project_client:
        # Create an agent and run user's request with function calls
        agent = project_client.agents.create_agent(
            model=os.environ["CHAT_MODEL"],
            name="function_calling_traced_agent",
            instructions="You are a helpful assistant",
            tools=functions.definitions,
        )
        print(f"Created agent, ID: {agent.id}")

        thread = project_client.agents.create_thread()
        print(f"Created thread, ID: {thread.id}")

        message = project_client.agents.create_message(
            thread_id=thread.id,
            role="user",
            content="Hello, what is the weather in New York?",
        )
        print(f"Created message, ID: {message.id}")

        run = project_client.agents.create_run(thread_id=thread.id, agent_id=agent.id)
        print(f"Created run, ID: {run.id}")

        while run.status in ["queued", "in_progress", "requires_action"]:
            time.sleep(1)
            run = project_client.agents.get_run(thread_id=thread.id, run_id=run.id)

            if run.status == "requires_action" and isinstance(run.required_action, SubmitToolOutputsAction):
                tool_calls = run.required_action.submit_tool_outputs.tool_calls
                if not tool_calls:
                    print("No tool calls provided - cancelling run")
                    project_client.agents.cancel_run(thread_id=thread.id, run_id=run.id)
                    break

                tool_outputs = []
                for tool_call in tool_calls:
                    if isinstance(tool_call, RequiredFunctionToolCall):
                        try:
                            output = functions.execute(tool_call)
                            tool_outputs.append(
                                ToolOutput(
                                    tool_call_id=tool_call.id,
                                    output=output,
                                )
                            )
                        except Exception as e:
                            print(f"Error executing tool_call {tool_call.id}: {e}")

                print(f"Tool outputs: {tool_outputs}")
                if tool_outputs:
                    project_client.agents.submit_tool_outputs_to_run(
                        thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs
                    )

            print(f"Current run status: {run.status}")

        print(f"Run completed with status: {run.status}")

        # Delete the agent when done
        # project_client.agents.delete_agent(agent.id)
        # print("Deleted agent")

        # Fetch and log all messages
        messages = project_client.agents.list_messages(thread_id=thread.id)
        print(f"Messages: {messages}")

`

This is the output that I get:

Created agent, ID: asst_Y18NvMBw4xuloAYyW5EOrDrp Created thread, ID: thread_ZvAtHWzthGlpu42HCYh7HbW4 Created message, ID: msg_VyiXmJB7S2ky3ndtUW0f43DE Created run, ID: run_7LAJDZYPGH0ujzbuyoWe8te2 Tool outputs: [{'tool_call_id': 'call_Dg1yHNgS73RoExw0RY7obmCm', 'output': '{"weather": "Sunny, 25\\u00b0C"}'}] Current run status: RunStatus.REQUIRES_ACTION Current run status: RunStatus.COMPLETED Run completed with status: RunStatus.COMPLETED Messages: {'object': 'list', 'data': [{'id': 'msg_K7dn62AkzlQOkzt62hA23cfg', 'object': 'thread.message', 'created_at': 1746178640, 'assistant_id': 'asst_Y18NvMBw4xuloAYyW5EOrDrp', 'thread_id': 'thread_ZvAtHWzthGlpu42HCYh7HbW4', 'run_id': 'run_7LAJDZYPGH0ujzbuyoWe8te2', 'role': 'assistant', 'content': [{'type': 'text', 'text': {'value': 'The weather in New York is sunny with a temperature of 25°C.', 'annotations': []}}], 'attachments': [], 'metadata': {}}, {'id': 'msg_VyiXmJB7S2ky3ndtUW0f43DE', 'object': 'thread.message', 'created_at': 1746178636, 'assistant_id': None, 'thread_id': 'thread_ZvAtHWzthGlpu42HCYh7HbW4', 'run_id': None, 'role': 'user', 'content': [{'type': 'text', 'text': {'value': 'Hello, what is the weather in New York?', 'annotations': []}}], 'attachments': [], 'metadata': {}}], 'first_id': 'msg_K7dn62AkzlQOkzt62hA23cfg', 'last_id': 'msg_VyiXmJB7S2ky3ndtUW0f43DE', 'has_more': False}

But I cant see the in/output in the Azure AI Foundry Tracing UI. I would appreciate your help.

#tracing @microsoftopensource

[spasegirl]

I followed this script: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/agents/sample_agents_functions_with_azure_monitor_tracing.py

[amynic]

Thanks for sharing the resource you are using to highlight a place we can make updates!

[nitya]

https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/agents/sample_agents_functions_with_azure_monitor_tracing.py

Happy to take a look at this - but finding that the script is no longer at that location. Do you have an updated script location I can look at? I am currently looking at Tracing samples so if you still have the issue, just share the relevant script or tutorial.

[leestott]

Hi @spasegirl

Troubleshooting Azure AI Foundry Tracing UI Issues

Below you will find a script to help diagnose and fix the issue with your Azure AI Foundry Tracing UI where your user inputs, assistant outputs, and tool execution details aren't visible.

Key Features of the script

1. Enhanced Tracing Instrumentation:

   • Adds explicit OpenTelemetry spans for each operation in your workflow
   • Includes detailed attributes for user inputs, assistant outputs, and tool calls
   • Creates a hierarchical span structure for better visibility in the UI

2. Multiple Telemetry Destinations:

   • Outputs trace information to both Application Insights and console
   • Ensures you can see trace data even if the UI is having issues

3. Explicit Function Tracing:

   • Uses the trace_function() decorator properly
   • Adds additional spans within the tool execution for maximum visibility

4. Detailed Logging and Diagnostics:

   • Logs each step of the process with detailed information
   • Verifies that the tracing context is valid and properly propagated
   • Includes troubleshooting guidance for common issues

How to Use the Script

1. Set the required environment variables in PowerShell:

   $env:AIPROJECT_CONNECTION_STRING = "your_connection_string" 
   $env:CHAT_MODEL = "your_model_name"  # e.g. "gpt-4" or your Azure OpenAI deployment name

2. Run the debug script:

   python debug.py

3. Check the console output for any errors or warnings in the tracing setup

4. Wait 2-5 minutes for the traces to be processed by Application Insights

5. Check the Azure AI Foundry Tracing UI for the new traces with the ID "ai_foundry_agent_debug_flow"

Common Issues and Solutions

Based on your description, here are likely causes and solutions:

1. Missing Trace Function Decorator:

   • The original code was missing proper @trace_function() decorators on user functions
   • The debug script adds this and creates explicit spans within the function

2. Incorrect Telemetry Configuration:

   • The debug script ensures telemetry is enabled with proper destinations
   • Configures Azure Monitor with the correct connection string

3. Missing Span Context:

   • Your original code may have been missing proper span context propagation
   • The debug script adds explicit span creation and attribute setting

4. Application Insights Configuration:

   • Verifies that Application Insights is properly configured
   • Logs the connection string status

If you're still having issues after running the debug script, please check:

• The Application Insights status in your Azure portal
• The service principal permissions for your AI Foundry project
• The region compatibility with tracing features
• Network restrictions that might be blocking telemetry data

The script provides extensive logging to help you identify exactly where the tracing pipeline might be breaking down.

"""
Debug Python script for Azure AI Foundry Tracing. This script helps troubleshoot issues with tracing UI not showing user inputs,
assistant outputs and tool execution details in Azure AI Foundry.
"""

import os
import time
import json
import sys
from typing import Any, Dict, List, Optional

from azure.ai.projects import AIProjectClient
from azure.ai.projects.telemetry import trace_function
from azure.ai.projects.models import (
    FunctionTool,
    RequiredFunctionToolCall,
    SubmitToolOutputsAction,
    ToolOutput
)
from azure.identity import DefaultAzureCredential
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
from azure.monitor.opentelemetry import configure_azure_monitor
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator

# Configure logging to see detailed diagnostic information
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# Define a sample function tool for weather lookup
@trace_function()
def get_weather(location: str) -> Dict[str, Any]:
    """Get the current weather for a location"""
    logger.info(f"Weather function called for location: {location}")
    
    # Create a span for this function call to ensure it's visible in tracing
    tracer = trace.get_tracer(__name__)
    with tracer.start_as_current_span("get_weather_function") as span:
        span.set_attribute("function.name", "get_weather")
        span.set_attribute("function.location", location)
        
        # Simulate API call
        weather_data = {"weather": "Sunny, 25°C"}
        span.set_attribute("function.result", json.dumps(weather_data))
        logger.info(f"Weather data returned: {weather_data}")
        
        return weather_data

# Collection of user functions
user_functions = {
    "get_weather": get_weather
}

def initialize_tracing():
    """Initialize and verify tracing configuration"""
    logger.info("Initializing tracing configuration...")
    
    # Check for connection string
    connection_string = os.environ.get("AIPROJECT_CONNECTION_STRING")
    if not connection_string:
        logger.error("AIPROJECT_CONNECTION_STRING environment variable not set")
        logger.info("Please set it with: $env:AIPROJECT_CONNECTION_STRING = 'your_connection_string'")
        return False
    
    try:
        # Create project client with specific context
        logger.info("Creating AI Project client...")
        project_client = AIProjectClient.from_connection_string(
            credential=DefaultAzureCredential(), 
            conn_str=connection_string
        )
        
        # Get Application Insights connection string
        logger.info("Getting Application Insights connection string...")
        app_insights_conn_string = project_client.telemetry.get_connection_string()
        if not app_insights_conn_string:
            logger.error("Application Insights is not enabled for this project")
            logger.info("Enable it via the 'Tracing' tab in your AI Foundry project page")
            return False
        
        logger.info("Configuring Azure Monitor with Application Insights connection string...")
        # Configure with proper exporters
        configure_azure_monitor(
            connection_string=app_insights_conn_string
        )
        
        # Enable telemetry with multiple destinations for maximum visibility
        logger.info("Enabling telemetry with multiple destinations...")
        # Enable to stdout for immediate visibility in console
        project_client.telemetry.enable(destination=sys.stdout)
        
        # Also enable general telemetry
        project_client.telemetry.enable()
        
        logger.info("Tracing configuration successful!")
        return project_client
    
    except Exception as e:
        logger.error(f"Error initializing tracing: {str(e)}")
        return False

def run_agent_with_enhanced_tracing():
    """Run the agent with enhanced tracing for debugging"""
    project_client = initialize_tracing()
    if not project_client:
        return

    # Get the model name from environment variable
    model = os.environ.get("CHAT_MODEL")
    if not model:
        logger.error("CHAT_MODEL environment variable not set")
        logger.info("Please set it with: $env:CHAT_MODEL = 'your_model_name'")
        return

    # Create tracer for explicit spans
    tracer = trace.get_tracer(__name__)

    # Define function tool
    functions = FunctionTool(functions=user_functions)

    # Run the entire flow within a parent span for better context
    with tracer.start_as_current_span("ai_foundry_agent_debug_flow") as parent_span:
        parent_span.set_attribute("debug.purpose", "troubleshoot_tracing_ui")
        
        try:
            with project_client:
                # Create agent with explicit span
                with tracer.start_as_current_span("create_agent") as span:
                    span.set_attribute("agent.instructions", "You are a helpful assistant")
                    span.set_attribute("agent.tools", json.dumps(functions.definitions))
                    
                    agent = project_client.agents.create_agent(
                        model=model,
                        name="debug_traced_agent",
                        instructions="You are a helpful assistant",
                        tools=functions.definitions,
                    )
                    span.set_attribute("agent.id", agent.id)
                    logger.info(f"Created agent, ID: {agent.id}")

                # Create thread with explicit span
                with tracer.start_as_current_span("create_thread") as span:
                    thread = project_client.agents.create_thread()
                    span.set_attribute("thread.id", thread.id)
                    logger.info(f"Created thread, ID: {thread.id}")

                # Create user message with explicit span
                with tracer.start_as_current_span("create_message") as span:
                    user_content = "Hello, what is the weather in New York?"
                    span.set_attribute("message.content", user_content)
                    span.set_attribute("message.role", "user")
                    
                    message = project_client.agents.create_message(
                        thread_id=thread.id,
                        role="user",
                        content=user_content,
                    )
                    span.set_attribute("message.id", message.id)
                    logger.info(f"Created message, ID: {message.id}")

                # Create run with explicit span
                with tracer.start_as_current_span("create_run") as span:
                    span.set_attribute("run.agent_id", agent.id)
                    span.set_attribute("run.thread_id", thread.id)
                    
                    run = project_client.agents.create_run(
                        thread_id=thread.id, 
                        agent_id=agent.id
                    )
                    span.set_attribute("run.id", run.id)
                    logger.info(f"Created run, ID: {run.id}")

                # Poll run status with explicit span for each iteration
                poll_iteration = 0
                while run.status in ["queued", "in_progress", "requires_action"]:
                    with tracer.start_as_current_span(f"poll_run_status_{poll_iteration}") as span:
                        span.set_attribute("run.id", run.id)
                        span.set_attribute("run.status", run.status)
                        span.set_attribute("poll.iteration", poll_iteration)
                        
                        time.sleep(1)
                        run = project_client.agents.get_run(
                            thread_id=thread.id, 
                            run_id=run.id
                        )
                        span.set_attribute("run.updated_status", run.status)
                        
                        if run.status == "requires_action" and isinstance(run.required_action, SubmitToolOutputsAction):
                            with tracer.start_as_current_span("process_tool_calls") as tool_span:
                                tool_calls = run.required_action.submit_tool_outputs.tool_calls
                                tool_span.set_attribute("tool_calls.count", len(tool_calls) if tool_calls else 0)
                                
                                if not tool_calls:
                                    logger.warn("No tool calls provided - cancelling run")
                                    project_client.agents.cancel_run(thread_id=thread.id, run_id=run.id)
                                    break

                                tool_outputs = []
                                for idx, tool_call in enumerate(tool_calls):
                                    if isinstance(tool_call, RequiredFunctionToolCall):
                                        with tracer.start_as_current_span(f"execute_tool_{idx}") as exec_span:
                                            exec_span.set_attribute("tool.call_id", tool_call.id)
                                            exec_span.set_attribute("tool.name", tool_call.function.name)
                                            exec_span.set_attribute("tool.arguments", tool_call.function.arguments)
                                            
                                            try:
                                                output = functions.execute(tool_call)
                                                exec_span.set_attribute("tool.output", json.dumps(output))
                                                
                                                tool_outputs.append(
                                                    ToolOutput(
                                                        tool_call_id=tool_call.id,
                                                        output=output,
                                                    )
                                                )
                                                exec_span.set_status(Status(StatusCode.OK))
                                            except Exception as e:
                                                error_msg = f"Error executing tool_call {tool_call.id}: {e}"
                                                logger.error(error_msg)
                                                exec_span.set_status(Status(StatusCode.ERROR, error_msg))
                                
                                tool_span.set_attribute("tool_outputs.count", len(tool_outputs))
                                logger.info(f"Tool outputs: {tool_outputs}")
                                
                                if tool_outputs:
                                    with tracer.start_as_current_span("submit_tool_outputs") as submit_span:
                                        submit_span.set_attribute("tool_outputs.count", len(tool_outputs))
                                        project_client.agents.submit_tool_outputs_to_run(
                                            thread_id=thread.id, 
                                            run_id=run.id, 
                                            tool_outputs=tool_outputs
                                        )
                        
                        logger.info(f"Current run status: {run.status}")
                        poll_iteration += 1

                with tracer.start_as_current_span("finalize_run") as span:
                    span.set_attribute("run.final_status", run.status)
                    logger.info(f"Run completed with status: {run.status}")

                # Fetch and log all messages with explicit span
                with tracer.start_as_current_span("fetch_messages") as span:
                    messages = project_client.agents.list_messages(thread_id=thread.id)
                    span.set_attribute("messages.count", len(messages.data))
                    
                    # Explicitly trace message contents for better visibility
                    for idx, msg in enumerate(messages.data):
                        with tracer.start_as_current_span(f"message_{idx}") as msg_span:
                            msg_span.set_attribute("message.id", msg.id)
                            msg_span.set_attribute("message.role", msg.role)
                            msg_span.set_attribute("message.thread_id", msg.thread_id)
                            
                            # Extract and trace message content
                            if msg.content and len(msg.content) > 0:
                                content = msg.content[0]
                                if content.get('type') == 'text':
                                    text_value = content.get('text', {}).get('value', '')
                                    msg_span.set_attribute("message.content", text_value)
                            
                    logger.info(f"Retrieved {len(messages.data)} messages from thread")
                    
                    # Print detailed message information
                    for msg in messages.data:
                        content_text = msg.content[0].get('text', {}).get('value', '') if msg.content else ''
                        logger.info(f"Message ID: {msg.id}, Role: {msg.role}, Content: {content_text[:100]}...")

                # Save tracing context
                with tracer.start_as_current_span("save_tracing_context") as span:
                    # Create a carrier dictionary to store the propagation headers
                    carrier = {}
                    TraceContextTextMapPropagator().inject(carrier)
                    span.set_attribute("tracing.context", json.dumps(carrier))
                    logger.info(f"Trace context: {carrier}")

        except Exception as e:
            logger.error(f"Error in agent flow: {str(e)}")
            parent_span.set_status(Status(StatusCode.ERROR, str(e)))

def main():
    """Main function to run the debug script"""
    logger.info("Starting Azure AI Foundry Tracing Debug Script...")
    
    # Check environment variables
    if not os.environ.get("AIPROJECT_CONNECTION_STRING"):
        logger.error("AIPROJECT_CONNECTION_STRING environment variable is not set")
        logger.info("Please set the environment variable with:")
        logger.info("$env:AIPROJECT_CONNECTION_STRING = 'your_connection_string'")
        return
    
    if not os.environ.get("CHAT_MODEL"):
        logger.error("CHAT_MODEL environment variable is not set")
        logger.info("Please set the environment variable with:")
        logger.info("$env:CHAT_MODEL = 'your_model_name'")
        return
    
    # Run the enhanced tracing flow
    run_agent_with_enhanced_tracing()
    
    # Verify tracing context is available and working
    current_span = trace.get_current_span()
    span_context = current_span.get_span_context()
    if span_context.is_valid:
        logger.info(f"Current trace ID: {span_context.trace_id}")
        logger.info(f"Current span ID: {span_context.span_id}")
    else:
        logger.warning("No valid span context found - tracing may not be working correctly")
    
    # Provide guidance on checking the tracing UI
    logger.info("\nDebugging completed. To check traces in Azure AI Foundry:")
    logger.info("1. Wait a few minutes for traces to be processed (typically 2-5 minutes)")
    logger.info("2. Go to your Azure AI Foundry project in the Azure portal")
    logger.info("3. Navigate to the Tracing tab")
    logger.info("4. Look for 'ai_foundry_agent_debug_flow' trace")
    logger.info("5. If you still don't see traces:")
    logger.info("   - Verify Application Insights is properly configured in your project")
    logger.info("   - Check that your service principal has appropriate permissions")
    logger.info("   - Ensure your deployment region supports tracing features")
    logger.info("   - Try setting explicit instrumentation key with:")
    logger.info("     $env:APPLICATIONINSIGHTS_CONNECTION_STRING = 'your_insights_connection_string'")

if __name__ == "__main__":
    main()

[amynic]

@spasegirl did @leestott's post above help you to find the issue?