Python: Function calling in Google connectors

python/semantic_kernel/connectors/ai/azure_ai_inference/azure_ai_inference_prompt_execution_settings.py

@@ -30,8 +30,15 @@ class AzureAIInferencePromptExecutionSettings(PromptExecutionSettings):
 class AzureAIInferenceChatPromptExecutionSettings(AzureAIInferencePromptExecutionSettings):
     """Azure AI Inference Chat Prompt Execution Settings."""
 
-    tools: list[dict[str, Any]] | None = Field(None, max_length=64)
-    tool_choice: str | None = None
+    tools: list[dict[str, Any]] | None = Field(
+        None,
+        max_length=64,
+        description="Do not set this manually. It is set by the service based on the function choice configuration.",
+    )
+    tool_choice: str | None = Field(
+        None,
+        description="Do not set this manually. It is set by the service based on the function choice configuration.",
+    )
 
 
 @experimental_class

python/semantic_kernel/connectors/ai/google/README.md

@@ -48,3 +48,7 @@ kernel.add_service(
 ```
 
 > Alternatively, you can use an .env file to store the model id and project id.
+
+## Why is there code that looks almost identical in the implementations on the two connectors
+
+The two connectors have very similar implementations, including the utils files. However, they are fundamentally different as they depend on different packages from Google. Although the namings of many types are identical, they are different types.

python/semantic_kernel/connectors/ai/google/google_ai/google_ai_prompt_execution_settings.py

@@ -35,8 +35,15 @@ class GoogleAITextPromptExecutionSettings(GoogleAIPromptExecutionSettings):
 class GoogleAIChatPromptExecutionSettings(GoogleAIPromptExecutionSettings):
     """Google AI Chat Prompt Execution Settings."""
 
-    tools: list[dict[str, Any]] | None = Field(None, max_length=64)
-    tool_choice: str | None = None
+    tools: list[dict[str, Any]] | None = Field(
+        None,
+        max_length=64,
+        description="Do not set this manually. It is set by the service based on the function choice configuration.",
+    )
+    tool_config: dict[str, Any] | None = Field(
+        None,
+        description="Do not set this manually. It is set by the service based on the function choice configuration.",
+    )
 
     @override
     def prepare_settings_dict(self, **kwargs) -> dict[str, Any]:
@@ -47,7 +54,7 @@ def prepare_settings_dict(self, **kwargs) -> dict[str, Any]:
         """
         settings_dict = super().prepare_settings_dict(**kwargs)
         settings_dict.pop("tools", None)
-        settings_dict.pop("tool_choice", None)
+        settings_dict.pop("tool_config", None)
 
         return settings_dict
 

python/semantic_kernel/connectors/ai/google/google_ai/services/google_ai_chat_completion.py

@@ -1,8 +1,10 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 
+import logging
 import sys
 from collections.abc import AsyncGenerator
+from functools import reduce
 from typing import TYPE_CHECKING, Any
 
 import google.generativeai as genai
@@ -18,12 +20,24 @@
 from semantic_kernel.connectors.ai.google.google_ai.services.utils import (
     finish_reason_from_google_ai_to_semantic_kernel,
     format_assistant_message,
+    format_tool_message,
     format_user_message,
+    update_settings_from_function_choice_configuration,
 )
-from semantic_kernel.connectors.ai.google.shared_utils import filter_system_message
+from semantic_kernel.connectors.ai.google.shared_utils import (
+    configure_function_choice_behavior,
+    filter_system_message,
+    format_gemini_function_name_to_kernel_function_fully_qualified_name,
+    invoke_function_calls,
+)
+from semantic_kernel.contents.function_call_content import FunctionCallContent
+from semantic_kernel.contents.streaming_chat_message_content import ITEM_TYPES as STREAMING_ITEM_TYPES
 from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
+from semantic_kernel.contents.streaming_text_content import StreamingTextContent
+from semantic_kernel.contents.text_content import TextContent
 from semantic_kernel.contents.utils.author_role import AuthorRole
 from semantic_kernel.contents.utils.finish_reason import FinishReason
+from semantic_kernel.kernel import Kernel
 
 if sys.version_info >= (3, 12):
     from typing import override  # pragma: no cover
@@ -33,12 +47,17 @@
 from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
 from semantic_kernel.connectors.ai.google.google_ai.google_ai_settings import GoogleAISettings
 from semantic_kernel.contents.chat_history import ChatHistory
-from semantic_kernel.contents.chat_message_content import ChatMessageContent
-from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
+from semantic_kernel.contents.chat_message_content import ITEM_TYPES, ChatMessageContent
+from semantic_kernel.exceptions.service_exceptions import (
+    ServiceInitializationError,
+    ServiceInvalidExecutionSettingsError,
+)
 
 if TYPE_CHECKING:
     from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
 
+logger: logging.Logger = logging.getLogger(__name__)
+
 
 class GoogleAIChatCompletion(GoogleAIBase, ChatCompletionClientBase):
     """Google AI Chat Completion Client."""
@@ -97,7 +116,40 @@ async def get_chat_message_contents(
         settings = self.get_prompt_execution_settings_from_settings(settings)
         assert isinstance(settings, GoogleAIChatPromptExecutionSettings)  # nosec
 
-        return await self._send_chat_request(chat_history, settings)
+        if (
+            settings.function_choice_behavior is None
+            or not settings.function_choice_behavior.auto_invoke_kernel_functions
+        ):
+            return await self._send_chat_request(chat_history, settings)
+
+        kernel = kwargs.get("kernel")
+        if not isinstance(kernel, Kernel):
+            raise ServiceInvalidExecutionSettingsError("Kernel is required for auto invoking functions.")
+
+        configure_function_choice_behavior(settings, kernel, update_settings_from_function_choice_configuration)
+
+        for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts):
+            completions = await self._send_chat_request(chat_history, settings)
+            chat_history.add_message(message=completions[0])
+            function_calls = [item for item in chat_history.messages[-1].items if isinstance(item, FunctionCallContent)]
+            if (fc_count := len(function_calls)) == 0:
+                return completions
+
+            results = await invoke_function_calls(
+                function_calls=function_calls,
+                chat_history=chat_history,
+                kernel=kernel,
+                arguments=kwargs.get("arguments", None),
+                function_call_count=fc_count,
+                request_index=request_index,
+                function_behavior=settings.function_choice_behavior,
+            )
+
+            if any(result.terminate for result in results if result is not None):
+                return completions
+        else:
+            # do a final call without auto function calling
+            return await self._send_chat_request(chat_history, settings)
 
     async def _send_chat_request(
         self, chat_history: ChatHistory, settings: GoogleAIChatPromptExecutionSettings
@@ -112,6 +164,8 @@ async def _send_chat_request(
         response: AsyncGenerateContentResponse = await model.generate_content_async(
             contents=self._prepare_chat_history_for_request(chat_history),
             generation_config=GenerationConfig(**settings.prepare_settings_dict()),
+            tools=settings.tools,
+            tool_config=settings.tool_config,
         )
 
         return [self._create_chat_message_content(response, candidate) for candidate in response.candidates]
@@ -133,10 +187,25 @@ def _create_chat_message_content(
         response_metadata = self._get_metadata_from_response(response)
         response_metadata.update(self._get_metadata_from_candidate(candidate))
 
+        items: list[ITEM_TYPES] = []
+        for idx, part in enumerate(candidate.content.parts):
+            if part.text:
+                items.append(TextContent(text=part.text, inner_content=response, metadata=response_metadata))
+            elif part.function_call:
+                items.append(
+                    FunctionCallContent(
+                        id=f"{part.function_call.name}_{idx!s}",
+                        name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
+                            part.function_call.name
+                        ),
+                        arguments={k: v for k, v in part.function_call.args.items()},
+                    )
+                )
+
         return ChatMessageContent(
             ai_model_id=self.ai_model_id,
             role=AuthorRole.ASSISTANT,
-            content=candidate.content.parts[0].text,
+            items=items,
             inner_content=response,
             finish_reason=finish_reason,
             metadata=response_metadata,
@@ -155,11 +224,68 @@ async def get_streaming_chat_message_contents(
         settings = self.get_prompt_execution_settings_from_settings(settings)
         assert isinstance(settings, GoogleAIChatPromptExecutionSettings)  # nosec
 
-        async_generator = self._send_chat_streaming_request(chat_history, settings)
+        if (
+            settings.function_choice_behavior is None
+            or not settings.function_choice_behavior.auto_invoke_kernel_functions
+        ):
+            # No auto invoke is required.
+            async_generator = self._send_chat_streaming_request(chat_history, settings)
+        else:
+            # Auto invoke is required.
+            async_generator = self._get_streaming_chat_message_contents_auto_invoke(chat_history, settings, **kwargs)
 
         async for messages in async_generator:
             yield messages
 
+    async def _get_streaming_chat_message_contents_auto_invoke(
+        self,
+        chat_history: ChatHistory,
+        settings: GoogleAIChatPromptExecutionSettings,
+        **kwargs: Any,
+    ) -> AsyncGenerator[list[StreamingChatMessageContent], Any]:
+        """Get streaming chat message contents from the Google AI service with auto invoking functions."""
+        kernel = kwargs.get("kernel")
+        if not isinstance(kernel, Kernel):
+            raise ServiceInvalidExecutionSettingsError("Kernel is required for auto invoking functions.")
+        if not settings.function_choice_behavior:
+            raise ServiceInvalidExecutionSettingsError(
+                "Function choice behavior is required for auto invoking functions."
+            )
+
+        configure_function_choice_behavior(settings, kernel, update_settings_from_function_choice_configuration)
+
+        for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts):
+            all_messages: list[StreamingChatMessageContent] = []
+            function_call_returned = False
+            async for messages in self._send_chat_streaming_request(chat_history, settings):
+                for message in messages:
+                    if message:
+                        all_messages.append(message)
+                        if any(isinstance(item, FunctionCallContent) for item in message.items):
+                            function_call_returned = True
+                yield messages
+
+            if not function_call_returned:
+                # Response doesn't contain any function calls. No need to proceed to the next request.
+                return
+
+            full_completion: StreamingChatMessageContent = reduce(lambda x, y: x + y, all_messages)
+            function_calls = [item for item in full_completion.items if isinstance(item, FunctionCallContent)]
+            chat_history.add_message(message=full_completion)
+
+            results = await invoke_function_calls(
+                function_calls=function_calls,
+                chat_history=chat_history,
+                kernel=kernel,
+                arguments=kwargs.get("arguments", None),
+                function_call_count=len(function_calls),
+                request_index=request_index,
+                function_behavior=settings.function_choice_behavior,
+            )
+
+            if any(result.terminate for result in results if result is not None):
+                return
+
     async def _send_chat_streaming_request(
         self,
         chat_history: ChatHistory,
@@ -175,6 +301,8 @@ async def _send_chat_streaming_request(
         response: AsyncGenerateContentResponse = await model.generate_content_async(
             contents=self._prepare_chat_history_for_request(chat_history),
             generation_config=GenerationConfig(**settings.prepare_settings_dict()),
+            tools=settings.tools,
+            tool_config=settings.tool_config,
             stream=True,
         )
 
@@ -200,11 +328,33 @@ def _create_streaming_chat_message_content(
         response_metadata = self._get_metadata_from_response(chunk)
         response_metadata.update(self._get_metadata_from_candidate(candidate))
 
+        items: list[STREAMING_ITEM_TYPES] = []
+        for idx, part in enumerate(candidate.content.parts):
+            if part.text:
+                items.append(
+                    StreamingTextContent(
+                        choice_index=candidate.index,
+                        text=part.text,
+                        inner_content=chunk,
+                        metadata=response_metadata,
+                    )
+                )
+            elif part.function_call:
+                items.append(
+                    FunctionCallContent(
+                        id=f"{part.function_call.name}_{idx!s}",
+                        name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
+                            part.function_call.name
+                        ),
+                        arguments={k: v for k, v in part.function_call.args.items()},
+                    )
+                )
+
         return StreamingChatMessageContent(
             ai_model_id=self.ai_model_id,
             role=AuthorRole.ASSISTANT,
             choice_index=candidate.index,
-            content=candidate.content.parts[0].text,
+            items=items,
             inner_content=chunk,
             finish_reason=finish_reason,
             metadata=response_metadata,
@@ -230,8 +380,8 @@ def _prepare_chat_history_for_request(
                 chat_request_messages.append(Content(role="user", parts=format_user_message(message)))
             elif message.role == AuthorRole.ASSISTANT:
                 chat_request_messages.append(Content(role="model", parts=format_assistant_message(message)))
-            else:
-                raise ValueError(f"Unsupported role: {message.role}")
+            elif message.role == AuthorRole.TOOL:
+                chat_request_messages.append(Content(role="function", parts=format_tool_message(message)))
 
         return chat_request_messages