Update javadoc for message package and docs for Gemini Multimodal support

markpollack · markpollack · commit 7f1570d72271 · 2024-03-11T19:14:53.000-04:00
diff --git a/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java b/models/spring-ai-vertex-ai-gemini/src/main/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClient.java
@@ -333,7 +333,7 @@ static List<Part> messageToGeminiParts(Message message, String systemContext) {
 
 			List<Part> parts = new ArrayList<>(List.of(textPart));
 
-			List<Part> mediaParts = userMessage.getMediaData()
+			List<Part> mediaParts = userMessage.getMedia()
 				.stream()
 				.map(mediaData -> PartMaker.fromMimeTypeAndData(mediaData.getMimeType().toString(),
 						mediaData.getData()))
diff --git a/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java b/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java
@@ -28,7 +28,7 @@
 import org.springframework.ai.chat.ChatResponse;
 import org.springframework.ai.chat.Generation;
 import org.springframework.ai.chat.messages.AssistantMessage;
-import org.springframework.ai.chat.messages.MediaData;
+import org.springframework.ai.chat.messages.Media;
 import org.springframework.ai.chat.messages.Message;
 import org.springframework.ai.chat.messages.UserMessage;
 import org.springframework.ai.chat.prompt.Prompt;
@@ -189,7 +189,7 @@ void multiModalityTest() throws IOException {
 		byte[] data = new ClassPathResource("/vertex.test.png").getContentAsByteArray();
 
 		var userMessage = new UserMessage("Explain what do you see o this picture?",
-				List.of(new MediaData(MimeTypeUtils.IMAGE_PNG, data)));
+				List.of(new Media(MimeTypeUtils.IMAGE_PNG, data)));
 
 		ChatResponse response = client.call(new Prompt(List.of(userMessage)));
 
@@ -208,7 +208,7 @@ void multiModalityTest() throws IOException {
 		// "https://storage.googleapis.com/github-repo/img/gemini/multimodality_usecases_overview/banana-apple.jpg";
 
 		// userMessage = new UserMessage("Explain what do you see o this picture?",
-		// List.of(new MediaData(MimeTypeDetector.getMimeType(imageUrl), imageUrl)));
+		// List.of(new Media(MimeTypeDetector.getMimeType(imageUrl), imageUrl)));
 		// response = client.call(new Prompt(List.of(userMessage)));
 
 		// assertThat(response.getResult().getOutput().getContent()).contains("bananas",
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/AbstractMessage.java
@@ -27,13 +27,20 @@
 import org.springframework.util.Assert;
 import org.springframework.util.StreamUtils;
 
+/**
+ * The AbstractMessage class is an abstract implementation of the Message interface. It
+ * provides a base implementation for message content, media attachments, properties, and
+ * message type.
+ *
+ * @see Message
+ */
 public abstract class AbstractMessage implements Message {
 
 	protected final MessageType messageType;
 
 	protected final String textContent;
 
-	protected final List<MediaData> mediaData;
+	protected final List<Media> mediaData;
 
 	/**
 	 * Additional options for the message to influence the response, not a generative map.
@@ -46,18 +53,17 @@ protected AbstractMessage(MessageType messageType, String content) {
 
 	protected AbstractMessage(MessageType messageType, String content, Map<String, Object> messageProperties) {
 		Assert.notNull(messageType, "Message type must not be null");
-		// Assert.notNull(content, "Content must not be null");
 		this.messageType = messageType;
 		this.textContent = content;
 		this.mediaData = new ArrayList<>();
 		this.properties = messageProperties;
 	}
 
-	protected AbstractMessage(MessageType messageType, String textContent, List<MediaData> mediaData) {
+	protected AbstractMessage(MessageType messageType, String textContent, List<Media> mediaData) {
 		this(messageType, textContent, mediaData, Map.of());
 	}
 
-	protected AbstractMessage(MessageType messageType, String textContent, List<MediaData> mediaData,
+	protected AbstractMessage(MessageType messageType, String textContent, List<Media> mediaData,
 			Map<String, Object> messageProperties) {
 
 		Assert.notNull(messageType, "Message type must not be null");
@@ -97,7 +103,7 @@ public String getContent() {
 	}
 
 	@Override
-	public List<MediaData> getMediaData() {
+	public List<Media> getMedia() {
 		return this.mediaData;
 	}
 
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/ChatMessage.java
@@ -17,6 +17,10 @@
 
 import java.util.Map;
 
+/**
+ * Represents a chat message in a chat application.
+ *
+ */
 public class ChatMessage extends AbstractMessage {
 
 	public ChatMessage(String role, String content) {
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/FunctionMessage.java
@@ -17,6 +17,10 @@
 
 import java.util.Map;
 
+/**
+ * The FunctionMessage class represents a message with a function content in a chat
+ * application.
+ */
 public class FunctionMessage extends AbstractMessage {
 
 	public FunctionMessage(String content) {
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Media.java
@@ -19,17 +19,22 @@
 import org.springframework.util.MimeType;
 
 /**
+ * The Media class represents the data and metadata of a media attachment in a message. It
+ * consists of a MIME type and the raw data.
+ *
+ * This class is used as a parameter in the constructor of the UserMessage class.
+ *
  * @author Christian Tzolov
+ * @since 0.8.1
  */
-public class MediaData {
+public class Media {
 
 	private final MimeType mimeType;
 
 	private final Object data;
 
-	public MediaData(MimeType mimeType, Object data) {
+	public Media(MimeType mimeType, Object data) {
 		Assert.notNull(mimeType, "MimeType must not be null");
-		// Assert.notNull(data, "Data must not be null");
 		this.mimeType = mimeType;
 		this.data = data;
 	}
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/Message.java
@@ -18,11 +18,19 @@
 import java.util.List;
 import java.util.Map;
 
+/**
+ * The Message interface represents a message that can be sent or received in a chat
+ * application. Messages can have content, media attachments, properties, and message
+ * types.
+ *
+ * @see Media
+ * @see MessageType
+ */
 public interface Message {
 
 	String getContent();
 
-	List<MediaData> getMediaData();
+	List<Media> getMedia();
 
 	Map<String, Object> getProperties();
 
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/MessageType.java
@@ -15,6 +15,10 @@
  */
 package org.springframework.ai.chat.messages;
 
+/**
+ * The MessageType enum represents the type of a message in a chat application. It can be
+ * one of the following: USER, ASSISTANT, SYSTEM, FUNCTION.
+ */
 public enum MessageType {
 
 	USER("user"),
diff --git a/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java b/spring-ai-core/src/main/java/org/springframework/ai/chat/messages/UserMessage.java
@@ -34,8 +34,8 @@ public UserMessage(Resource resource) {
 		super(MessageType.USER, resource);
 	}
 
-	public UserMessage(String textContent, List<MediaData> mediaDataList) {
-		super(MessageType.USER, textContent, mediaDataList);
+	public UserMessage(String textContent, List<Media> mediaList) {
+		super(MessageType.USER, textContent, mediaList);
 	}
 
 	@Override
diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/clients/vertexai-gemini-chat.adoc
@@ -107,6 +107,26 @@ You can register custom Java functions with the VertexAiGeminiChatClient and hav
 This is a powerful technique to connect the LLM capabilities with external tools and APIs.
 Read more about xref:api/clients/functions/vertexai-gemini-chat-functions.adoc[Vertex AI Gemini Function Calling].
 
+=== Multimodal Example
+Multimodality refers to a model's ability to simultaneously understand and process information from various sources, including text, images, audio, and other data formats. This paradigm represents a significant advancement in AI models.
+
+Google's Gemini AI models support this capability by comprehending and integrating text, code, audio, images, and video. For more details, refer to the blog post [Introducing Gemini](https://blog.google/technology/ai/google-gemini-ai/#introducing-gemini).
+
+Spring AI's `Message` interface supports multimodal AI models by introducing the Media type.
+This type contains data and information about media attachments in messages, using Spring's `org.springframework.util.MimeType` and a `java.lang.Object` for the raw media data.
+
+Below is a simple code example extracted from [VertexAiGeminiChatClientIT.java](https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java), demonstrating the combination of user text with an image.
+
+
+[source,java]
+----
+byte[] data = new ClassPathResource("/vertex-test.png").getContentAsByteArray();
+
+var userMessage = new UserMessage("Explain what do you see o this picture?",
+        List.of(new Media(MimeTypeUtils.IMAGE_PNG, data)));
+
+ChatResponse response = chatClient.call(new Prompt(List.of(userMessage)));
+----
 
 === Sample Controller (Auto-configuration)
 
@@ -195,6 +215,3 @@ ChatResponse response = chatClient.call(
 The `VertexAiGeminiChatOptions` provides the configuration information for the chat requests.
 The `VertexAiGeminiChatOptions.Builder` is fluent options builder.
 
-=== Multi-modal
-
-https://github.com/spring-projects/spring-ai/blob/main/models/spring-ai-vertex-ai-gemini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatClientIT.java[VertexAiGeminiChatClientIT.java]
diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/prompt.adoc
@@ -45,16 +45,19 @@ public class Prompt {
 
 === Message
 
-The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, and a categorization known as `MessageType`. The interface is defined as follows:
+The `Message` interface encapsulates a textual message, a collection of attributes as a `Map`, a categorization known as `MessageType`, and a list of media objects for those models that are multimodal.
+The interface is defined as follows:
 
 ```java
 public interface Message {
 
-   String getContent();
+	String getContent();
 
-   Map<String, Object> getProperties();
+	List<Media> getMedia();
 
-   MessageType getMessageType();
+	Map<String, Object> getProperties();
+
+	MessageType getMessageType();
 
 }
 ```

Original file line number	Diff line number	Diff line change
`@@ -34,8 +34,8 @@ public UserMessage(Resource resource) {`
`34`	`34`	`super(MessageType.USER, resource);`
`35`	`35`	`}`
`36`	`36`
`37`		`- public UserMessage(String textContent, List<MediaData> mediaDataList) {`
`38`		`- super(MessageType.USER, textContent, mediaDataList);`
	`37`	`+ public UserMessage(String textContent, List<Media> mediaList) {`
	`38`	`+ super(MessageType.USER, textContent, mediaList);`
`39`	`39`	`}`
`40`	`40`
`41`	`41`	`@Override`