Enhance ChatClientRequestSpec with sealed Prompt input

 - When ChatClientRequestSpec#prompt(Prompt) is used, unseal the prompt instance.
   Convert the last message instance (if user message) into spec#user and spec#media
   and add the remaining messages (excluding the last) to the spec#messages.
   Add the prompt#options to the spec#options.
 - Improve DefaultChatClient to handle UserMessage media and content separately.
 - Update AbstractToolCallSupport to use new hasToolCalls() method.
 - Add hasToolCalls() method to AssistantMessage.
 - Enhance ChatClientTest with additional test cases for media handling.
 - Disable Groq and Nvidia integration tests due to rate limiting and credit requirements.

Author: tzolov

models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/proxy/GroqWithOpenAiChatModelIT.java

@@ -66,6 +66,7 @@
 
 @SpringBootTest(classes = GroqWithOpenAiChatModelIT.Config.class)
 @EnabledIfEnvironmentVariable(named = "GROQ_API_KEY", matches = ".+")
+@Disabled("Due to rate limiting it is hard to run it in one go")
 class GroqWithOpenAiChatModelIT {
 
 	private static final Logger logger = LoggerFactory.getLogger(OpenAiChatModelIT.class);

models/spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/proxy/NvidiaWithOpenAiChatModelIT.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 import org.slf4j.Logger;
@@ -61,6 +62,7 @@
  */
 @SpringBootTest(classes = NvidiaWithOpenAiChatModelIT.Config.class)
 @EnabledIfEnvironmentVariable(named = "NVIDIA_API_KEY", matches = ".+")
+@Disabled("Requires NVIDIA credits")
 class NvidiaWithOpenAiChatModelIT {
 
 	private static final Logger logger = LoggerFactory.getLogger(NvidiaWithOpenAiChatModelIT.class);

spring-ai-core/src/main/java/org/springframework/ai/chat/client/DefaultChatClient.java

@@ -42,6 +42,7 @@
 import org.springframework.ai.chat.client.observation.ChatClientObservationDocumentation;
 import org.springframework.ai.chat.client.observation.DefaultChatClientObservationConvention;
 import org.springframework.ai.chat.messages.Message;
+import org.springframework.ai.chat.messages.MessageType;
 import org.springframework.ai.chat.messages.SystemMessage;
 import org.springframework.ai.chat.messages.UserMessage;
 import org.springframework.ai.chat.model.ChatModel;
@@ -104,11 +105,33 @@ public ChatClientRequestSpec prompt(String content) {
 	public ChatClientRequestSpec prompt(Prompt prompt) {
 
 		DefaultChatClientRequestSpec spec = new DefaultChatClientRequestSpec(this.defaultChatClientRequest);
-		spec.messages(prompt.getInstructions());
+
+		// Options
 		if (prompt.getOptions() != null) {
 			spec.options(prompt.getOptions());
 		}
 
+		// Messages
+		List<Message> messages = prompt.getInstructions();
+
+		if (!CollectionUtils.isEmpty(messages)) {
+			var lastMessage = messages.get(messages.size() - 1);
+			if (lastMessage.getMessageType() == MessageType.USER) {
+				// Unzip the last message
+				var userMessage = (UserMessage) lastMessage;
+				if (StringUtils.hasText(userMessage.getContent())) {
+					spec.user(lastMessage.getContent());
+				}
+				var media = userMessage.getMedia();
+				if (!CollectionUtils.isEmpty(media)) {
+					spec.user(u -> u.media(media.toArray(new Media[media.size()])));
+				}
+				messages = messages.subList(0, messages.size() - 1);
+			}
+		}
+
+		spec.messages(messages);
+
 		return spec;
 	}
 

spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AssistantMessage.java

@@ -20,6 +20,7 @@
 import java.util.Objects;
 
 import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
 
 /**
  * Lets the generative know the content was generated as a response to the user. This role
@@ -52,15 +53,14 @@ public AssistantMessage(String content, Map<String, Object> properties, List<Too
 		this.toolCalls = toolCalls;
 	}
 
-	@Override
-	public String getContent() {
-		return this.textContent;
-	}
-
 	public List<ToolCall> getToolCalls() {
 		return this.toolCalls;
 	}
 
+	public boolean hasToolCalls() {
+		return !CollectionUtils.isEmpty(this.toolCalls);
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		if (this == o)

spring-ai-core/src/main/java/org/springframework/ai/chat/model/AbstractToolCallSupport.java

@@ -230,7 +230,7 @@ protected boolean isToolCall(ChatResponse chatResponse, Set<String> toolCallFini
 	protected boolean isToolCall(Generation generation, Set<String> toolCallFinishReasons) {
 		var finishReason = (generation.getMetadata().getFinishReason() != null)
 				? generation.getMetadata().getFinishReason() : "";
-		return !CollectionUtils.isEmpty(generation.getOutput().getToolCalls()) && toolCallFinishReasons.stream()
+		return generation.getOutput().hasToolCalls() && toolCallFinishReasons.stream()
 			.map(s -> s.toLowerCase())
 			.toList()
 			.contains(finishReason.toLowerCase());

spring-ai-core/src/test/java/org/springframework/ai/chat/client/ChatClientTest.java

@@ -38,6 +38,7 @@
 import org.springframework.ai.chat.model.ChatResponse;
 import org.springframework.ai.chat.model.Generation;
 import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.model.Media;
 import org.springframework.ai.model.function.FunctionCallingOptions;
 import org.springframework.ai.model.function.FunctionCallingOptionsBuilder;
 import org.springframework.ai.model.function.FunctionCallingOptionsBuilder.PortableFunctionCallingOptions;
@@ -67,14 +68,15 @@ private String join(Flux<String> fluxContent) {
 	@Test
 	public void defaultSystemText() {
 
-		when(chatModel.call(promptCaptor.capture())).thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
+		when(chatModel.call(promptCaptor.capture()))
+			.thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
 
-		when(chatModel.stream(promptCaptor.capture()))
-			.thenReturn(Flux.generate(() -> new ChatResponse(List.of(new Generation(new AssistantMessage("response")))), (state, sink) -> {
-				sink.next(state);
-				sink.complete();
-				return state;
-			}));
+		when(chatModel.stream(promptCaptor.capture())).thenReturn(Flux.generate(
+				() -> new ChatResponse(List.of(new Generation(new AssistantMessage("response")))), (state, sink) -> {
+					sink.next(state);
+					sink.complete();
+					return state;
+				}));
 
 		var chatClient = ChatClient.builder(chatModel).defaultSystem("Default system text").build();
 
@@ -114,14 +116,15 @@ public void defaultSystemText() {
 	@Test
 	public void defaultSystemTextLambda() {
 
-		when(chatModel.call(promptCaptor.capture())).thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
+		when(chatModel.call(promptCaptor.capture()))
+			.thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
 
-		when(chatModel.stream(promptCaptor.capture()))
-			.thenReturn(Flux.generate(() -> new ChatResponse(List.of(new Generation(new AssistantMessage("response")))), (state, sink) -> {
-				sink.next(state);
-				sink.complete();
-				return state;
-			}));
+		when(chatModel.stream(promptCaptor.capture())).thenReturn(Flux.generate(
+				() -> new ChatResponse(List.of(new Generation(new AssistantMessage("response")))), (state, sink) -> {
+					sink.next(state);
+					sink.complete();
+					return state;
+				}));
 
 		var chatClient = ChatClient.builder(chatModel)
 			.defaultSystem(s -> s.text("Default system text {param1}, {param2}")
@@ -438,10 +441,9 @@ public void defaultUserText() {
 	@Test
 	public void simpleUserPromptAsString() {
 		when(chatModel.call(promptCaptor.capture()))
-				.thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
+			.thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
 
-		assertThat(ChatClient.builder(chatModel).build().prompt("User prompt").call().content())
-				.isEqualTo("response");
+		assertThat(ChatClient.builder(chatModel).build().prompt("User prompt").call().content()).isEqualTo("response");
 
 		Message userMessage = promptCaptor.getValue().getInstructions().get(0);
 		assertThat(userMessage.getContent()).isEqualTo("User prompt");
@@ -466,13 +468,17 @@ public void simpleUserPromptObject() throws MalformedURLException {
 		when(chatModel.call(promptCaptor.capture()))
 			.thenReturn(new ChatResponse(List.of(new Generation(new AssistantMessage("response")))));
 
-		UserMessage message = new UserMessage("User prompt");
+		var media = new Media(MimeTypeUtils.IMAGE_JPEG, new DefaultResourceLoader().getResource("classpath:/bikes.json"));
+
+		UserMessage message = new UserMessage("User prompt", List.of(media));
 		Prompt prompt = new Prompt(message);
 		assertThat(ChatClient.builder(chatModel).build().prompt(prompt).call().content()).isEqualTo("response");
 
+		assertThat(promptCaptor.getValue().getInstructions()).hasSize(1);
 		Message userMessage = promptCaptor.getValue().getInstructions().get(0);
-		assertThat(userMessage.getContent()).isEqualTo("User prompt");
 		assertThat(userMessage.getMessageType()).isEqualTo(MessageType.USER);
+		assertThat(userMessage.getContent()).isEqualTo("User prompt");
+		assertThat(((UserMessage) userMessage).getMedia()).hasSize(1);
 	}
 
 	@Test
@@ -527,7 +533,7 @@ public void complexCall() throws MalformedURLException {
 		assertThat(userMessage.getMedia().iterator().next().getMimeType()).isEqualTo(MimeTypeUtils.IMAGE_PNG);
 		assertThat(userMessage.getMedia().iterator().next().getData())
 			.isEqualTo("https://docs.spring.io/spring-ai/reference/1.0-SNAPSHOT/_images/multimodal.test.png");
-		
+
 		FunctionCallingOptions runtieOptions = (FunctionCallingOptions) promptCaptor.getValue().getOptions();
 
 		assertThat(runtieOptions.getFunctions()).containsExactly("function1");