Adding support for transcriptions ( and OpenAI Whisper model)

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiTranscriptionClient.java

@@ -0,0 +1,168 @@
+package org.springframework.ai.openai;/*
+										* Copyright 2023-2023 the original author or authors.
+										*
+										* Licensed under the Apache License, Version 2.0 (the "License");
+										* you may not use this file except in compliance with the License.
+										* You may obtain a copy of the License at
+										*
+										*      https://www.apache.org/licenses/LICENSE-2.0
+										*
+										* Unless required by applicable law or agreed to in writing, software
+										* distributed under the License is distributed on an "AS IS" BASIS,
+										* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+										* See the License for the specific language governing permissions and
+										* limitations under the License.
+										*/
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.chat.ChatOptions;
+import org.springframework.ai.chat.metadata.RateLimit;
+import org.springframework.ai.model.ModelOptionsUtils;
+import org.springframework.ai.openai.api.OpenAiApi;
+import org.springframework.ai.openai.api.OpenAiApi.OpenAiApiException;
+import org.springframework.ai.openai.metadata.OpenAiTranscriptionResponseMetadata;
+import org.springframework.ai.openai.metadata.support.OpenAiResponseHeaderExtractor;
+import org.springframework.ai.transcription.*;
+import org.springframework.core.io.Resource;
+import org.springframework.http.ResponseEntity;
+import org.springframework.retry.support.RetryTemplate;
+import org.springframework.util.Assert;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * {@link TranscriptionClient} implementation for {@literal OpenAI} backed by
+ * {@link OpenAiApi}.
+ *
+ * @author Michael Lavelle
+ * @see TranscriptionClient
+ * @see OpenAiApi
+ */
+public class OpenAiTranscriptionClient implements TranscriptionClient {
+
+	private final Logger logger = LoggerFactory.getLogger(getClass());
+
+	private OpenAiTranscriptionOptions defaultOptions = OpenAiTranscriptionOptions.builder()
+		.withModel("whisper-1")
+		.withTemperature(0.7f)
+		.build();
+
+	public final RetryTemplate retryTemplate = RetryTemplate.builder()
+		.maxAttempts(10)
+		.retryOn(OpenAiApiException.class)
+		.exponentialBackoff(Duration.ofMillis(2000), 5, Duration.ofMillis(3 * 60000))
+		.build();
+
+	private final OpenAiApi openAiApi;
+
+	public OpenAiTranscriptionClient(OpenAiApi openAiApi) {
+		Assert.notNull(openAiApi, "OpenAiApi must not be null");
+		this.openAiApi = openAiApi;
+	}
+
+	public OpenAiTranscriptionClient withDefaultOptions(OpenAiTranscriptionOptions options) {
+		this.defaultOptions = options;
+		return this;
+	}
+
+	@Override
+	public TranscriptionResponse call(TranscriptionRequest request) {
+
+		return this.retryTemplate.execute(ctx -> {
+			Resource audioResource = request.getInstructions();
+
+			MultiValueMap<String, Object> requestBody = createRequestBody(request);
+
+			boolean jsonResponse = !requestBody.containsKey("response_format")
+					|| requestBody.get("response_format").contains("json");
+
+			if (jsonResponse) {
+
+				ResponseEntity<OpenAiApi.Transcription> transcriptionEntity = this.openAiApi
+					.transcriptionEntityJson(requestBody);
+
+				var transcription = transcriptionEntity.getBody();
+
+				if (transcription == null) {
+					logger.warn("No transcription returned for request: {}", audioResource);
+					return new TranscriptionResponse(null);
+				}
+
+				Transcript transcript = new Transcript(transcription.text());
+
+				RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity);
+
+				return new TranscriptionResponse(transcript,
+						OpenAiTranscriptionResponseMetadata.from(transcriptionEntity.getBody())
+							.withRateLimit(rateLimits));
+
+			}
+			else {
+				ResponseEntity<String> transcriptionEntity = this.openAiApi.transcriptionEntityText(requestBody);
+
+				var transcription = transcriptionEntity.getBody();
+
+				if (transcription == null) {
+					logger.warn("No transcription returned for request: {}", audioResource);
+					return new TranscriptionResponse(null);
+				}
+
+				Transcript transcript = new Transcript(transcription);
+
+				RateLimit rateLimits = OpenAiResponseHeaderExtractor.extractAiResponseHeaders(transcriptionEntity);
+
+				return new TranscriptionResponse(transcript,
+						OpenAiTranscriptionResponseMetadata.from(transcriptionEntity.getBody())
+							.withRateLimit(rateLimits));
+
+			}
+
+		});
+	}
+
+	private MultiValueMap<String, Object> createRequestBody(TranscriptionRequest transcriptionRequest) {
+
+		OpenAiApi.TranscriptionRequest request = new OpenAiApi.TranscriptionRequest();
+
+		if (this.defaultOptions != null) {
+			request = ModelOptionsUtils.merge(request, this.defaultOptions, OpenAiApi.TranscriptionRequest.class);
+		}
+
+		if (transcriptionRequest.getOptions() != null) {
+			if (transcriptionRequest.getOptions() instanceof TranscriptionOptions runtimeOptions) {
+				OpenAiTranscriptionOptions updatedRuntimeOptions = ModelOptionsUtils.copyToTarget(runtimeOptions,
+						TranscriptionOptions.class, OpenAiTranscriptionOptions.class);
+				request = ModelOptionsUtils.merge(updatedRuntimeOptions, request, OpenAiApi.TranscriptionRequest.class);
+			}
+			else {
+				throw new IllegalArgumentException("Prompt options are not of type TranscriptionOptions: "
+						+ transcriptionRequest.getOptions().getClass().getSimpleName());
+			}
+		}
+		MultiValueMap<String, Object> requestBody = new LinkedMultiValueMap<>();
+		if (request.responseFormat() != null) {
+			requestBody.add("response_format", request.responseFormat().type());
+		}
+		if (request.prompt() != null) {
+			requestBody.add("prompt", request.prompt());
+		}
+		if (request.temperature() != null) {
+			requestBody.add("temperature", request.temperature());
+		}
+		if (request.language() != null) {
+			requestBody.add("language", request.language());
+		}
+		if (request.model() != null) {
+			requestBody.add("model", request.model());
+		}
+		if (transcriptionRequest.getInstructions() != null) {
+			requestBody.add("file", transcriptionRequest.getInstructions());
+		}
+		return requestBody;
+	}
+
+}

models/spring-ai-openai/src/main/java/org/springframework/ai/openai/OpenAiTranscriptionOptions.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2024-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.ai.openai;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.springframework.ai.openai.api.OpenAiApi.TranscriptionRequest.ResponseFormat;
+import org.springframework.ai.transcription.TranscriptionOptions;
+
+/**
+ * @author Michael Lavelle
+ */
+@JsonInclude(Include.NON_NULL)
+public class OpenAiTranscriptionOptions implements TranscriptionOptions {
+
+	// @formatter:off
+	/**
+	 * ID of the model to use.
+	 */
+	private @JsonProperty("model") String model;
+
+	/**
+	 * An object specifying the format that the model must output. Setting to { "type":
+	 * "json_object" } enables JSON mode, which guarantees the message the model generates is valid JSON.
+	 */
+	private @JsonProperty("response_format") ResponseFormat responseFormat;
+
+	private @JsonProperty("prompt") String prompt;
+
+	private @JsonProperty("language") String language;
+
+	/**
+	 * What sampling temperature to use, between 0 and 1. Higher values like 0.8 will make the output
+	 * more random, while lower values like 0.2 will make it more focused and deterministic.
+	 */
+	private @JsonProperty("temperature") Float temperature = 0.8f;
+
+
+	public static Builder builder() {
+		return new Builder();
+	}
+
+	public static class Builder {
+
+		protected OpenAiTranscriptionOptions options;
+
+		public Builder() {
+			this.options = new OpenAiTranscriptionOptions();
+		}
+
+		public Builder(OpenAiTranscriptionOptions options) {
+			this.options = options;
+		}
+
+		public Builder withModel(String model) {
+			this.options.model = model;
+			return this;
+		}
+
+		public Builder withLanguage(String language) {
+			this.options.language = language;
+			return this;
+		}
+
+		public Builder withPrompt(String prompt) {
+			this.options.prompt = prompt;
+			return this;
+		}
+
+		public Builder withResponseFormat(ResponseFormat responseFormat) {
+			this.options.responseFormat = responseFormat;
+			return this;
+		}
+
+		public Builder withTemperature(Float temperature) {
+			this.options.temperature = temperature;
+			return this;
+		}
+
+		public OpenAiTranscriptionOptions build() {
+			return this.options;
+		}
+
+	}
+
+	public String getModel() {
+		return this.model;
+	}
+
+	public void setModel(String model) {
+		this.model = model;
+	}
+
+	public String getLanguage() {
+		return this.language;
+	}
+
+	public void setLanguage(String language) {
+		this.language = language;
+	}
+
+	public String getPrompt() {
+		return this.prompt;
+	}
+
+	public void setPrompt(String prompt) {
+		this.prompt = prompt;
+	}
+
+	public Float getTemperature() {
+		return this.temperature;
+	}
+
+	public void setTemperature(Float temperature) {
+		this.temperature = temperature;
+	}
+
+
+	public ResponseFormat getResponseFormat() {
+		return this.responseFormat;
+	}
+
+	public void setResponseFormat(ResponseFormat responseFormat) {
+		this.responseFormat = responseFormat;
+	}
+
+
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((model == null) ? 0 : model.hashCode());
+		result = prime * result + ((prompt == null) ? 0 : prompt.hashCode());
+		result = prime * result + ((language == null) ? 0 : language.hashCode());
+		result = prime * result + ((responseFormat == null) ? 0 : responseFormat.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		OpenAiTranscriptionOptions other = (OpenAiTranscriptionOptions) obj;
+		if (this.model == null) {
+			if (other.model != null)
+				return false;
+		}
+		else if (!model.equals(other.model))
+			return false;
+		if (this.prompt == null) {
+			if (other.prompt != null)
+				return false;
+		}
+		else if (!this.prompt.equals(other.prompt))
+			return false;
+		if (this.language == null) {
+			if (other.language != null)
+				return false;
+		}
+		else if (!this.language.equals(other.language))
+			return false;
+		if (this.responseFormat == null) {
+			if (other.responseFormat != null)
+				return false;
+		}
+		else if (!this.responseFormat.equals(other.responseFormat))
+			return false;
+		return true;
+	}
+}