Add Tool Call Accuracy Evaluator Bugbash Notebook

sdk/ai/azure-ai-agents/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 # Release History
 
+## 1.1.0b5 (Unreleased)
+
+### Bugs Fixed
+
+- `AgentsResponseFormatOption`, `MessageInputContent`, `MessageAttachmentToolDefinition`, `AgentsToolChoiceOption` are now public.
+- Fixed issues where the `runs.create_and_process` API call did not correctly handle the `AzureAISearchTool`, `FileSearchTool`, and `CodeInterpreterTool` when specified in the toolset parameter.
+
 ## 1.1.0b4 (2025-07-11)
 
 ### Features Added
@@ -12,7 +19,7 @@ the thread. Default value is None.
 
 ### Bugs Fixed
 
-- `_AgentsClientOperationsMixin` now it is private.
+- `_AgentsClientOperationsMixin` is now private.
 
 ### Sample updates
 

sdk/ai/azure-ai-agents/azure/ai/agents/_version.py

@@ -6,4 +6,4 @@
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 
-VERSION = "1.1.0b4"
+VERSION = "1.1.0b5"

sdk/ai/azure-ai-agents/azure/ai/agents/aio/operations/_patch.py

@@ -49,7 +49,7 @@
 
 if TYPE_CHECKING:
     # pylint: disable=unused-import,ungrouped-imports
-    from ... import _types
+    from ... import types as _types
 
 JSON = MutableMapping[str, Any]  # pylint: disable=unsubscriptable-object
 _Unset: Any = object()
@@ -193,8 +193,10 @@ async def create(  # pylint: disable=arguments-differ
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -356,8 +358,10 @@ async def create(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -497,9 +501,10 @@ async def create_and_process(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or
-         ~azure.ai.agents.models.AgentsApiResponseFormatMode or
-         ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -525,6 +530,7 @@ async def create_and_process(
             additional_instructions=additional_instructions,
             additional_messages=additional_messages,
             tools=toolset.definitions if toolset else None,
+            tool_resources=toolset.resources if toolset else None,
             temperature=temperature,
             top_p=top_p,
             max_prompt_tokens=max_prompt_tokens,
@@ -673,8 +679,10 @@ async def stream(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -782,8 +790,10 @@ async def stream(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -961,8 +971,10 @@ async def stream(  # pyright: ignore[reportInconsistentOverload]
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -1339,7 +1351,7 @@ async def upload(  # pylint: disable=arguments-differ
         """Uploads a file for use by other operations.
 
         :keyword file_path: The path to the file to upload.
-        :type file_path: str
+        :paramtype file_path: str
         :keyword purpose: The intended purpose of the uploaded file. Known values are: "assistants", "assistants_output", and "vision".
         :paramtype purpose: str or ~azure.ai.agents.models.FilePurpose
         :return: FileInfo. The FileInfo is compatible with MutableMapping
@@ -1500,7 +1512,7 @@ async def upload_and_poll(
         """Uploads a file for use by other operations.
 
         :keyword file_path: The path to the file to upload.
-        :type file_path: str
+        :paramtype file_path: str
         :keyword purpose: Known values are: "assistants", "assistants_output", and "vision".
         :paramtype purpose: str or ~azure.ai.agents.models.FilePurpose
         :keyword polling_interval: Time to wait before polling for the status of the uploaded file. Default value

sdk/ai/azure-ai-agents/azure/ai/agents/models/_patch.py

@@ -92,7 +92,7 @@
 from ._models import ThreadMessage as ThreadMessageGenerated
 from ._models import MessageAttachment as MessageAttachmentGenerated
 
-from .. import _types
+from .. import types as _types
 
 
 logger = logging.getLogger(__name__)

sdk/ai/azure-ai-agents/azure/ai/agents/operations/_patch.py

@@ -48,7 +48,7 @@
 
 if TYPE_CHECKING:
     # pylint: disable=unused-import,ungrouped-imports
-    from .. import _types
+    from .. import types as _types
 
 JSON = MutableMapping[str, Any]  # pylint: disable=unsubscriptable-object
 _Unset: Any = object()
@@ -196,8 +196,10 @@ def create(  # pylint: disable=arguments-differ
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -363,8 +365,10 @@ def create(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -505,9 +509,10 @@ def create_and_process(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or
-         ~azure.ai.agents.models.AgentsApiResponseFormatMode or
-         ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -533,6 +538,7 @@ def create_and_process(
             additional_instructions=additional_instructions,
             additional_messages=additional_messages,
             tools=toolset.definitions if toolset else None,
+            tool_resources=toolset.resources if toolset else None,
             temperature=temperature,
             top_p=top_p,
             max_prompt_tokens=max_prompt_tokens,
@@ -685,8 +691,10 @@ def stream(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -798,8 +806,10 @@ def stream(
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -982,8 +992,10 @@ def stream(  # pyright: ignore[reportInconsistentOverload]
         :keyword response_format: Specifies the format that the model must output. Is one of the
          following types: str, Union[str, "_models.AgentsApiResponseFormatMode"],
          AgentsApiResponseFormat Default value is None.
-        :paramtype response_format: str or str or ~azure.ai.agents.models.AgentsApiResponseFormatMode
-         or ~azure.ai.agents.models.AgentsApiResponseFormat
+        :paramtype response_format: Optional[Union[str,
+                               ~azure.ai.agents.models.AgentsApiResponseFormatMode,
+                               ~azure.ai.agents.models.AgentsApiResponseFormat,
+                               ~azure.ai.agents.models.ResponseFormatJsonSchemaType]]
         :keyword parallel_tool_calls: If ``true`` functions will run in parallel during tool use.
          Default value is None.
         :paramtype parallel_tool_calls: bool
@@ -1372,7 +1384,7 @@ def upload(  # pylint: disable=arguments-differ
         """Uploads a file for use by other operations.
 
         :keyword file_path: The path to the file to upload.
-        :type file_path: str
+        :paramtype file_path: str
         :keyword purpose: Known values are: "assistants", "assistants_output", and "vision".
         :paramtype purpose: str or ~azure.ai.agents.models.FilePurpose
         :return: FileInfo. The FileInfo is compatible with MutableMapping
@@ -1533,7 +1545,7 @@ def upload_and_poll(
         """Uploads a file for use by other operations.
 
         :keyword file_path: The path to the file on the local filesystem to upload.
-        :type file_path: str
+        :paramtype file_path: str
         :keyword purpose: Known values are: "assistants", "assistants_output", and "vision".
         :paramtype purpose: str or ~azure.ai.agents.models.FilePurpose
         :keyword polling_interval: Time to wait before polling for the status of the uploaded file. Default value

sdk/ai/azure-ai-agents/azure/ai/agents/types/__init__.py

@@ -0,0 +1,19 @@
+# pylint: disable=line-too-long,useless-suppression,too-many-lines
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+
+from azure.ai.agents._types import (
+    AgentsResponseFormatOption,
+    MessageInputContent,
+    MessageAttachmentToolDefinition,
+    AgentsToolChoiceOption,
+)
+
+__all__ = [
+    "AgentsResponseFormatOption",
+    "MessageInputContent",
+    "MessageAttachmentToolDefinition",
+    "AgentsToolChoiceOption",
+]