Add full support or Vertex AI Gemini and Azure OpenAI function calling

 - Extend the Spring AI Message with getMediaData() : List<MediaData>
   MediaData is a pair of MimeType and data of type Object.
   Message#getContent() return text only.

 - VertexAI Gemini Support
   - implement VertexAiGeiminChatClient for ChatClient and StreamingChat client and support for MediaData content.
     add IT tests for Chat, Streaming and Multimodality
   - add Auto-configuration + ITs
   - add Gemini Spring Boot starter.
   - add clients and boot starters to the Spring AI BOM.
   - add Anotra documentation for the Gemini chat client.
   - update gemini to latest 26.33.0 BOM.
   - add vertex ai gemini dependencies to the BOM.
   - add Vertex AI Gemini API Function Calling support
   - add Gemini API Function Calling Streaming support
   - add vertex ai gemini function calling documentation
   - factor out the Function Calling functionality into common abstraction used by OpenAI, Azure and Gemini.
   - group the VertexAI documentation under a common parent
   - add PortableFunctionCallingOption that implements FunctionCallingOptions and ChatOptions and provide builder for it.
   - remove some deprecated code.
   - allow authorization with GoogleCredentials form json file.
   - add AOT support for VertexAI Gemini.
   - move legacy Vertex AI into VertexAI PaLM2.
   - better handling for empty chat responses.
   - update the Gemini version to latest 26.33.0. This required lifting the protobuf-java to 3.25.2 as well.
   - fix a bug for handling System messages with Gemini.

- Implement Azure OpenAI Function Calling
  Uses the same the common abstractions used by OpenAI and Gemini: FunctionCallingOptions and AbstractFunctionCallSupport

Author: tzolov

models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatClient.java

@@ -17,35 +17,49 @@
 package org.springframework.ai.azure.openai;
 
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import com.azure.ai.openai.OpenAIClient;
 import com.azure.ai.openai.models.ChatChoice;
 import com.azure.ai.openai.models.ChatCompletions;
+import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall;
+import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition;
 import com.azure.ai.openai.models.ChatCompletionsOptions;
+import com.azure.ai.openai.models.ChatCompletionsToolCall;
+import com.azure.ai.openai.models.ChatCompletionsToolDefinition;
 import com.azure.ai.openai.models.ChatRequestAssistantMessage;
 import com.azure.ai.openai.models.ChatRequestMessage;
 import com.azure.ai.openai.models.ChatRequestSystemMessage;
+import com.azure.ai.openai.models.ChatRequestToolMessage;
 import com.azure.ai.openai.models.ChatRequestUserMessage;
+import com.azure.ai.openai.models.ChatResponseMessage;
+import com.azure.ai.openai.models.CompletionsFinishReason;
 import com.azure.ai.openai.models.ContentFilterResultsForPrompt;
+import com.azure.ai.openai.models.FunctionDefinition;
+import com.azure.core.util.BinaryData;
 import com.azure.core.util.IterableStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import reactor.core.publisher.Flux;
 
 import org.springframework.ai.azure.openai.metadata.AzureOpenAiChatResponseMetadata;
 import org.springframework.ai.chat.ChatClient;
-import org.springframework.ai.chat.prompt.ChatOptions;
 import org.springframework.ai.chat.ChatResponse;
 import org.springframework.ai.chat.Generation;
 import org.springframework.ai.chat.StreamingChatClient;
 import org.springframework.ai.chat.messages.Message;
 import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
 import org.springframework.ai.chat.metadata.PromptMetadata;
 import org.springframework.ai.chat.metadata.PromptMetadata.PromptFilterMetadata;
+import org.springframework.ai.chat.prompt.ChatOptions;
 import org.springframework.ai.chat.prompt.Prompt;
 import org.springframework.ai.model.ModelOptionsUtils;
+import org.springframework.ai.model.function.AbstractFunctionCallSupport;
+import org.springframework.ai.model.function.FunctionCallbackContext;
 import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
 
 /**
  * {@link ChatClient} implementation for {@literal Microsoft Azure AI} backed by
@@ -58,7 +72,9 @@
  * @see ChatClient
  * @see com.azure.ai.openai.OpenAIClient
  */
-public class AzureOpenAiChatClient implements ChatClient, StreamingChatClient {
+public class AzureOpenAiChatClient
+		extends AbstractFunctionCallSupport<ChatRequestMessage, ChatCompletionsOptions, ChatCompletions>
+		implements ChatClient, StreamingChatClient {
 
 	private static final String DEFAULT_MODEL = "gpt-35-turbo";
 
@@ -82,6 +98,12 @@ public AzureOpenAiChatClient(OpenAIClient microsoftOpenAiClient) {
 	}
 
 	public AzureOpenAiChatClient(OpenAIClient microsoftOpenAiClient, AzureOpenAiChatOptions options) {
+		this(microsoftOpenAiClient, options, null);
+	}
+
+	public AzureOpenAiChatClient(OpenAIClient microsoftOpenAiClient, AzureOpenAiChatOptions options,
+			FunctionCallbackContext functionCallbackContext) {
+		super(functionCallbackContext);
 		Assert.notNull(microsoftOpenAiClient, "com.azure.ai.openai.OpenAIClient must not be null");
 		Assert.notNull(options, "AzureOpenAiChatOptions must not be null");
 		this.openAIClient = microsoftOpenAiClient;
@@ -100,7 +122,7 @@ public AzureOpenAiChatClient withDefaultOptions(AzureOpenAiChatOptions defaultOp
 	}
 
 	public AzureOpenAiChatOptions getDefaultOptions() {
-		return defaultOptions;
+		return this.defaultOptions;
 	}
 
 	@Override
@@ -111,7 +133,10 @@ public ChatResponse call(Prompt prompt) {
 
 		logger.trace("Azure ChatCompletionsOptions: {}", options);
 
-		ChatCompletions chatCompletions = this.openAIClient.getChatCompletions(options.getModel(), options);
+		ChatCompletions chatCompletions = this.callWithFunctionSupport(options);
+
+		// ChatCompletions chatCompletions =
+		// this.openAIClient.getChatCompletions(options.getModel(), options);
 
 		logger.trace("Azure ChatCompletions: {}", chatCompletions);
 
@@ -154,6 +179,8 @@ public Flux<ChatResponse> stream(Prompt prompt) {
 	 */
 	ChatCompletionsOptions toAzureChatCompletionsOptions(Prompt prompt) {
 
+		Set<String> functionsForThisRequest = new HashSet<>();
+
 		List<ChatRequestMessage> azureMessages = prompt.getInstructions()
 			.stream()
 			.map(this::fromSpringAiMessage)
@@ -167,6 +194,10 @@ ChatCompletionsOptions toAzureChatCompletionsOptions(Prompt prompt) {
 			// options = ModelOptionsUtils.merge(options, this.defaultOptions,
 			// ChatCompletionsOptions.class);
 			options = merge(options, this.defaultOptions);
+
+			Set<String> defaultEnabledFunctions = this.handleFunctionCallbackConfigurations(this.defaultOptions,
+					!IS_RUNTIME_CALL);
+			functionsForThisRequest.addAll(defaultEnabledFunctions);
 		}
 
 		if (prompt.getOptions() != null) {
@@ -178,16 +209,43 @@ ChatCompletionsOptions toAzureChatCompletionsOptions(Prompt prompt) {
 				// options = ModelOptionsUtils.merge(runtimeOptions, options,
 				// ChatCompletionsOptions.class);
 				options = merge(updatedRuntimeOptions, options);
+
+				Set<String> promptEnabledFunctions = this.handleFunctionCallbackConfigurations(updatedRuntimeOptions,
+						IS_RUNTIME_CALL);
+				functionsForThisRequest.addAll(promptEnabledFunctions);
+
 			}
 			else {
 				throw new IllegalArgumentException("Prompt options are not of type ChatCompletionsOptions:"
 						+ prompt.getOptions().getClass().getSimpleName());
 			}
 		}
 
+		// Add the enabled functions definitions to the request's tools parameter.
+
+		if (!CollectionUtils.isEmpty(functionsForThisRequest)) {
+			List<ChatCompletionsFunctionToolDefinition> tools = this.getFunctionTools(functionsForThisRequest);
+			List<ChatCompletionsToolDefinition> tools2 = tools.stream()
+				.map(t -> ((ChatCompletionsToolDefinition) t))
+				.toList();
+			options.setTools(tools2);
+		}
+
 		return options;
 	}
 
+	private List<ChatCompletionsFunctionToolDefinition> getFunctionTools(Set<String> functionNames) {
+		return this.resolveFunctionCallbacks(functionNames).stream().map(functionCallback -> {
+
+			FunctionDefinition functionDefinition = new FunctionDefinition(functionCallback.getName());
+			functionDefinition.setDescription(functionCallback.getDescription());
+			BinaryData parameters = BinaryData
+				.fromObject(ModelOptionsUtils.jsonToMap(functionCallback.getInputTypeSchema()));
+			functionDefinition.setParameters(parameters);
+			return new ChatCompletionsFunctionToolDefinition(functionDefinition);
+		}).toList();
+	}
+
 	private ChatRequestMessage fromSpringAiMessage(Message message) {
 
 		switch (message.getMessageType()) {
@@ -281,6 +339,8 @@ private ChatCompletionsOptions merge(AzureOpenAiChatOptions springAiOptions, Cha
 		ChatCompletionsOptions mergedAzureOptions = new ChatCompletionsOptions(azureOptions.getMessages());
 		mergedAzureOptions = merge(azureOptions, mergedAzureOptions);
 
+		mergedAzureOptions.setStream(azureOptions.isStream());
+
 		if (springAiOptions.getMaxTokens() != null) {
 			mergedAzureOptions.setMaxTokens(springAiOptions.getMaxTokens());
 		}
@@ -324,6 +384,8 @@ private ChatCompletionsOptions merge(AzureOpenAiChatOptions springAiOptions, Cha
 		return mergedAzureOptions;
 	}
 
+	// https://github.com/Azure/azure-sdk-for-java/blob/azure-ai-openai_1.0.0-beta.6/sdk/openai/azure-ai-openai/src/samples/java/com/azure/ai/openai/usage/GetChatCompletionsToolCallSample.java
+
 	private ChatCompletionsOptions merge(ChatCompletionsOptions fromOptions, ChatCompletionsOptions toOptions) {
 
 		if (fromOptions == null) {
@@ -367,4 +429,68 @@ private ChatCompletionsOptions merge(ChatCompletionsOptions fromOptions, ChatCom
 		return mergedOptions;
 	}
 
+	@Override
+	protected ChatCompletionsOptions doCreateToolResponseRequest(ChatCompletionsOptions previousRequest,
+			ChatRequestMessage responseMessage, List<ChatRequestMessage> conversationHistory) {
+
+		// Every tool-call item requires a separate function call and a response (TOOL)
+		// message.
+		for (ChatCompletionsToolCall toolCall : ((ChatRequestAssistantMessage) responseMessage).getToolCalls()) {
+
+			var functionName = ((ChatCompletionsFunctionToolCall) toolCall).getFunction().getName();
+			String functionArguments = ((ChatCompletionsFunctionToolCall) toolCall).getFunction().getArguments();
+
+			if (!this.functionCallbackRegister.containsKey(functionName)) {
+				throw new IllegalStateException("No function callback found for function name: " + functionName);
+			}
+
+			String functionResponse = this.functionCallbackRegister.get(functionName).call(functionArguments);
+
+			// Add the function response to the conversation.
+			conversationHistory.add(new ChatRequestToolMessage(functionResponse, toolCall.getId()));
+		}
+
+		// Recursively call chatCompletionWithTools until the model doesn't call a
+		// functions anymore.
+		ChatCompletionsOptions newRequest = new ChatCompletionsOptions(conversationHistory);
+
+		newRequest = merge(previousRequest, newRequest);
+
+		return newRequest;
+	}
+
+	@Override
+	protected List<ChatRequestMessage> doGetUserMessages(ChatCompletionsOptions request) {
+		return request.getMessages();
+	}
+
+	@Override
+	protected ChatRequestMessage doGetToolResponseMessage(ChatCompletions response) {
+		ChatResponseMessage responseMessage = response.getChoices().get(0).getMessage();
+		ChatRequestAssistantMessage assistantMessage = new ChatRequestAssistantMessage("");
+		assistantMessage.setToolCalls(responseMessage.getToolCalls());
+		return assistantMessage;
+	}
+
+	@Override
+	protected ChatCompletions doChatCompletion(ChatCompletionsOptions request) {
+		return this.openAIClient.getChatCompletions(request.getModel(), request);
+	}
+
+	@Override
+	protected boolean isToolFunctionCall(ChatCompletions chatCompletions) {
+
+		if (chatCompletions == null || CollectionUtils.isEmpty(chatCompletions.getChoices())) {
+			return false;
+		}
+
+		var choice = chatCompletions.getChoices().get(0);
+
+		if (choice == null || choice.getFinishReason() == null) {
+			return false;
+		}
+
+		return choice.getFinishReason() == CompletionsFinishReason.TOOL_CALLS;
+	}
+
 }

models/spring-ai-azure-openai/src/main/java/org/springframework/ai/azure/openai/AzureOpenAiChatOptions.java

@@ -16,15 +16,22 @@
 
 package org.springframework.ai.azure.openai;
 
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import org.springframework.ai.chat.prompt.ChatOptions;
+import org.springframework.ai.model.function.FunctionCallback;
+import org.springframework.ai.model.function.FunctionCallingOptions;
+import org.springframework.boot.context.properties.NestedConfigurationProperty;
+import org.springframework.util.Assert;
 
 /**
  * The configuration information for a chat completions request. Completions support a
@@ -34,7 +41,7 @@
  * @author Christian Tzolov
  */
 @JsonInclude(Include.NON_NULL)
-public class AzureOpenAiChatOptions implements ChatOptions {
+public class AzureOpenAiChatOptions implements FunctionCallingOptions, ChatOptions {
 
 	/**
 	 * The maximum number of tokens to generate.
@@ -121,6 +128,32 @@ public class AzureOpenAiChatOptions implements ChatOptions {
 	@JsonProperty(value = "model")
 	private String model;
 
+	/**
+	 * OpenAI Tool Function Callbacks to register with the ChatClient. For Prompt Options
+	 * the functionCallbacks are automatically enabled for the duration of the prompt
+	 * execution. For Default Options the functionCallbacks are registered but disabled by
+	 * default. Use the enableFunctions to set the functions from the registry to be used
+	 * by the ChatClient chat completion requests.
+	 */
+	@NestedConfigurationProperty
+	@JsonIgnore
+	private List<FunctionCallback> functionCallbacks = new ArrayList<>();
+
+	/**
+	 * List of functions, identified by their names, to configure for function calling in
+	 * the chat completion requests. Functions with those names must exist in the
+	 * functionCallbacks registry. The {@link #functionCallbacks} from the PromptOptions
+	 * are automatically enabled for the duration of the prompt execution.
+	 *
+	 * Note that function enabled with the default options are enabled for all chat
+	 * completion requests. This could impact the token count and the billing. If the
+	 * functions is set in a prompt options, then the enabled functions are only active
+	 * for the duration of this prompt execution.
+	 */
+	@NestedConfigurationProperty
+	@JsonIgnore
+	private Set<String> functions = new HashSet<>();
+
 	public static Builder builder() {
 		return new Builder();
 	}
@@ -187,6 +220,23 @@ public Builder withUser(String user) {
 			return this;
 		}
 
+		public Builder withFunctionCallbacks(List<FunctionCallback> functionCallbacks) {
+			this.options.functionCallbacks = functionCallbacks;
+			return this;
+		}
+
+		public Builder withFunctions(Set<String> functionNames) {
+			Assert.notNull(functionNames, "Function names must not be null");
+			this.options.functions = functionNames;
+			return this;
+		}
+
+		public Builder withFunction(String functionName) {
+			Assert.hasText(functionName, "Function name must not be empty");
+			this.options.functions.add(functionName);
+			return this;
+		}
+
 		public AzureOpenAiChatOptions build() {
 			return this.options;
 		}
@@ -289,4 +339,24 @@ public void setTopK(Integer topK) {
 		throw new UnsupportedOperationException("Unimplemented method 'setTopK'");
 	}
 
+	@Override
+	public List<FunctionCallback> getFunctionCallbacks() {
+		return this.functionCallbacks;
+	}
+
+	@Override
+	public void setFunctionCallbacks(List<FunctionCallback> functionCallbacks) {
+		this.functionCallbacks = functionCallbacks;
+	}
+
+	@Override
+	public Set<String> getFunctions() {
+		return this.functions;
+	}
+
+	@Override
+	public void setFunctions(Set<String> functions) {
+		this.functions = functions;
+	}
+
 }