Allowing model options to be coniguratble on TranscriptionRequest, making tests more robust to non-deterministic effects, and minor polishing

Author: michaellavelle

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiTranscriptionClient.java

@@ -75,24 +75,52 @@ public TranscriptionResponse call(TranscriptionRequest request) {
 		return this.retryTemplate.execute(ctx -> {
 			Resource audioResource = request.getInstructions();
 
-			MultiValueMap<String, Object> reqyestBody = createRequestBody(request);
+			MultiValueMap<String, Object> requestBody = createRequestBody(request);
 
-			ResponseEntity<OpenAiApi.Transcription> transcriptionEntity = this.openAiApi
-				.transcriptionEntity(reqyestBody);
+			boolean jsonResponse = !requestBody.containsKey("response_format")
+					|| requestBody.get("response_format").contains("json");
 
-			var transcription = transcriptionEntity.getBody();
+			if (jsonResponse) {
+
+				ResponseEntity<OpenAiApi.Transcription> transcriptionEntity = this.openAiApi
+					.transcriptionEntityJson(requestBody);
+
+				var transcription = transcriptionEntity.getBody();
+
+				if (transcription == null) {
+					logger.warn("No transcription returned for request: {}", audioResource);
+					return new TranscriptionResponse(null);
+				}
+
+				Transcript transcript = new Transcript(transcription.text());
+
+				RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity);
+
+				return new TranscriptionResponse(transcript,
+						OpenAiTranscriptionResponseMetadata.from(transcriptionEntity.getBody())
+							.withRateLimit(rateLimits));
 
-			if (transcription == null) {
-				logger.warn("No transcription returned for request: {}", audioResource);
-				return new TranscriptionResponse(null);
 			}
+			else {
+				ResponseEntity<String> transcriptionEntity = this.openAiApi.transcriptionEntityText(requestBody);
+
+				var transcription = transcriptionEntity.getBody();
+
+				if (transcription == null) {
+					logger.warn("No transcription returned for request: {}", audioResource);
+					return new TranscriptionResponse(null);
+				}
 
-			Transcript transcript = new Transcript(transcription.text());
+				Transcript transcript = new Transcript(transcription);
 
-			RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity);
+				RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity);
+
+				return new TranscriptionResponse(transcript,
+						OpenAiTranscriptionResponseMetadata.from(transcriptionEntity.getBody())
+							.withRateLimit(rateLimits));
+
+			}
 
-			return new TranscriptionResponse(transcript,
-					OpenAiTranscriptionResponseMetadata.from(transcriptionEntity.getBody()).withRateLimit(rateLimits));
 		});
 	}
 

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiTranscriptionOptions.java

@@ -16,19 +16,12 @@
 
 package org.springframework.ai.openai;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.springframework.ai.chat.ChatOptions;
-import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ResponseFormat;
-import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoice;
-import org.springframework.ai.openai.api.OpenAiApi.FunctionTool;
+import org.springframework.ai.openai.api.OpenAiApi.TranscriptionRequest.ResponseFormat;
 import org.springframework.ai.transcription.TranscriptionOptions;
 
-import java.util.List;
-import java.util.Map;
-
 /**
  * @author Michael Lavelle
  */

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/api/OpenAiApi.java

@@ -722,9 +722,9 @@ public ResponseEntity<ChatCompletion> chatCompletionEntity(ChatCompletionRequest
 	 * Creates a model response for the given transcription.
 	 *
 	 * @param transcriptionRequest The transcription request.
-	 * @return Entity response with {@link MultiValueMap} as a body and HTTP status code and headers.
+	 * @return Entity response with {@link Transcription} as a body and HTTP status code and headers.
 	 */
-	public ResponseEntity<Transcription> transcriptionEntity(MultiValueMap<String, Object> transcriptionRequest) {
+	public ResponseEntity<Transcription> transcriptionEntityJson(MultiValueMap<String, Object> transcriptionRequest) {
 
 		Assert.notNull(transcriptionRequest, "The request body can not be null.");
 
@@ -735,6 +735,24 @@ public ResponseEntity<Transcription> transcriptionEntity(MultiValueMap<String, O
 				.toEntity(Transcription.class);
 	}
 
+	/**
+	 * Creates a model response for the given transcription.
+	 *
+	 * @param transcriptionRequest The transcription request.
+	 * @return Entity response with {@link String} as a body and HTTP status code and headers.
+	 */
+	public ResponseEntity<String> transcriptionEntityText(MultiValueMap<String, Object> transcriptionRequest) {
+
+		Assert.notNull(transcriptionRequest, "The request body can not be null.");
+
+		return this.multipartFormEncodingRestClient.post()
+				.uri("/v1/audio/transcriptions")
+				.body(transcriptionRequest)
+				.accept(MediaType.TEXT_PLAIN)
+				.retrieve()
+				.toEntity(String.class);
+	}
+
 	/**
 	 * Creates a streaming chat response for the given chat conversation.
 	 *

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/metadata/OpenAiTranscriptionResponseMetadata.java

@@ -39,6 +39,12 @@ public static OpenAiTranscriptionResponseMetadata from(OpenAiApi.Transcription r
 		return transcriptionResponseMetadata;
 	}
 
+	public static OpenAiTranscriptionResponseMetadata from(String result) {
+		Assert.notNull(result, "OpenAI Transcription must not be null");
+		OpenAiTranscriptionResponseMetadata transcriptionResponseMetadata = new OpenAiTranscriptionResponseMetadata();
+		return transcriptionResponseMetadata;
+	}
+
 	@Nullable
 	private RateLimit rateLimit;
 

models/spring-ai-openai/src/test/java/org/springframework/ai/openai/transcription/OpenAiTranscriptionClientIT.java

@@ -3,7 +3,11 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 import org.springframework.ai.openai.OpenAiTestConfiguration;
+import org.springframework.ai.openai.OpenAiTranscriptionOptions;
+import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.ai.openai.testutils.AbstractIT;
+import org.springframework.ai.transcription.TranscriptionOptions;
+import org.springframework.ai.transcription.TranscriptionOptionsBuilder;
 import org.springframework.ai.transcription.TranscriptionRequest;
 import org.springframework.ai.transcription.TranscriptionResponse;
 import org.springframework.beans.factory.annotation.Value;
@@ -21,11 +25,27 @@ class OpenAiTranscriptionClientIT extends AbstractIT {
 
 	@Test
 	void transcriptionTest() {
-		TranscriptionRequest transcriptionRequest = new TranscriptionRequest(audioFile);
+		TranscriptionOptions transcriptionOptions = TranscriptionOptionsBuilder.builder().withTemperature(0f).build();
+		TranscriptionRequest transcriptionRequest = new TranscriptionRequest(audioFile, transcriptionOptions);
 		TranscriptionResponse response = openAiTranscriptionClient.call(transcriptionRequest);
 		assertThat(response.getResults()).hasSize(1);
-		assertThat(response.getResults().get(0).getOutput()).isEqualTo(
-				"And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.");
+		assertThat(response.getResults().get(0).getOutput().toLowerCase().contains("fellow")).isTrue();
+	}
+
+	@Test
+	void transcriptionTestWithOptions() {
+		OpenAiApi.TranscriptionRequest.ResponseFormat responseFormat = new OpenAiApi.TranscriptionRequest.ResponseFormat(
+				"vtt");
+		TranscriptionOptions transcriptionOptions = OpenAiTranscriptionOptions.builder()
+			.withLanguage("en")
+			.withPrompt("Ask not this, but ask that")
+			.withTemperature(0f)
+			.withResponseFormat(responseFormat)
+			.build();
+		TranscriptionRequest transcriptionRequest = new TranscriptionRequest(audioFile, transcriptionOptions);
+		TranscriptionResponse response = openAiTranscriptionClient.call(transcriptionRequest);
+		assertThat(response.getResults()).hasSize(1);
+		assertThat(response.getResults().get(0).getOutput().toLowerCase().contains("fellow")).isTrue();
 	}
 
 }

spring-ai-core/src/main/java/org/springframework/ai/transcription/TranscriptionRequest.java

@@ -14,6 +14,11 @@ public TranscriptionRequest(Resource audioResource) {
 		this.audioResource = audioResource;
 	}
 
+	public TranscriptionRequest(Resource audioResource, ModelOptions modelOptions) {
+		this.audioResource = audioResource;
+		this.modelOptions = modelOptions;
+	}
+
 	@Override
 	public Resource getInstructions() {
 		return audioResource;

spring-ai-spring-boot-autoconfigure/src/test/java/org/springframework/ai/autoconfigure/openai/OpenAiPropertiesTests.java

@@ -20,6 +20,7 @@
 
 import org.junit.jupiter.api.Test;
 
+import org.springframework.ai.openai.api.OpenAiApi;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ResponseFormat;
 import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest.ToolChoice;
 import org.springframework.ai.openai.api.OpenAiApi.FunctionTool.Type;
@@ -366,7 +367,7 @@ public void transcriptionOptionsTest() {
 				assertThat(transcriptionProperties.getOptions().getLanguage()).isEqualTo("en");
 				assertThat(transcriptionProperties.getOptions().getPrompt()).isEqualTo("Er, yes, I think so");
 				assertThat(transcriptionProperties.getOptions().getResponseFormat())
-					.isEqualTo(new ResponseFormat("json"));
+					.isEqualTo(new OpenAiApi.TranscriptionRequest.ResponseFormat("json"));
 				assertThat(transcriptionProperties.getOptions().getTemperature()).isEqualTo(0.55f);
 			});
 	}