Whisper Audio API Implementation (#29)

Added
- Audio Transcription endpoint implemented
- Audio translation endpoint implemented
- Whisper sample project created
Changed
- Extension method for adding files merged into AddFile that takes contentType param
- Sample project paths in package.json updated

Author: srcnalt

Runtime/DataTypes.cs

@@ -118,6 +118,31 @@ public struct ChatMessage
     }
     #endregion
 
+    #region Audio Transcriptions Data Types
+
+    public class CreateAudioRequestBase
+    {
+        public string File { get; set; }
+        public string Model { get; set; }
+        public string Prompt { get; set; }
+        public string ResponseFormat { get; set; } = AudioResponseFormat.Json;
+        public float? Temperature { get; set; } = 0;
+    }
+    
+    public class CreateAudioTranscriptionsRequest: CreateAudioRequestBase
+    {
+        public string Language { get; set; }
+    }
+    
+    public class CreateAudioTranslationRequest: CreateAudioRequestBase { }
+    
+    public struct CreateAudioResponse: IResponse
+    {
+        public ApiError Error { get; set; }
+        public string Text { get; set; }
+    }
+    #endregion
+
     #region Completions API Data Types
     public sealed class CreateCompletionRequest
     {
@@ -364,7 +389,16 @@ public static class ImageResponseFormat
         public const string Url = "url";
         public const string Base64Json = "b64_json";
     }
-
+    
+    public static class AudioResponseFormat
+    {
+        public const string Json = "json";
+        public const string Text = "text";
+        public const string Srt = "srt";
+        public const string VerboseJson = "verbose_json";
+        public const string Vtt = "vtt";
+    }
+    
     public static class ModerationModel
     {
         public const string Stable = "text-moderation-stable";

Runtime/OpenAIApi.cs

@@ -5,6 +5,7 @@
 using System.Threading.Tasks;
 using UnityEngine.Networking;
 using System.Collections.Generic;
+using System.IO;
 using Newtonsoft.Json.Serialization;
 
 namespace OpenAI
@@ -166,6 +167,11 @@ public async Task<CreateCompletionResponse> CreateCompletion(CreateCompletionReq
             return await DispatchRequest<CreateCompletionResponse>(path, UnityWebRequest.kHttpVerbPOST, payload);
         }
 
+        /// <summary>
+        ///     Creates a chat completion request as in ChatGPT.
+        /// </summary>
+        /// <param name="request">See <see cref="CreateChatCompletionRequest"/></param>
+        /// <returns>See <see cref="CreateChatCompletionResponse"/></returns>
         public async Task<CreateChatCompletionResponse> CreateChatCompletion(CreateChatCompletionRequest request)
         {
             var path = $"{BASE_PATH}/chat/completions";
@@ -207,8 +213,8 @@ public async Task<CreateImageResponse> CreateImageEdit(CreateImageEditRequest re
             var path = $"{BASE_PATH}/images/edits";
 
             var form = new List<IMultipartFormSection>();
-            form.AddImage(request.Image, "image");
-            form.AddImage(request.Mask, "mask");
+            form.AddFile(request.Image, "image", "image/png");
+            form.AddFile(request.Mask, "mask", "image/png");
             form.AddValue(request.Prompt, "prompt");
             form.AddValue(request.N, "n");
             form.AddValue(request.Size, "size");
@@ -227,7 +233,7 @@ public async Task<CreateImageResponse> CreateImageVariation(CreateImageVariation
             var path = $"{BASE_PATH}/images/variations";
 
             var form = new List<IMultipartFormSection>();
-            form.AddImage(request.Image, "image");
+            form.AddFile(request.Image, "image", "image/png");
             form.AddValue(request.N, "n");
             form.AddValue(request.Size, "size");
             form.AddValue(request.ResponseFormat, "response_format");
@@ -247,6 +253,45 @@ public async Task<CreateEmbeddingsResponse> CreateEmbeddings(CreateEmbeddingsReq
             var payload = CreatePayload(request);
             return await DispatchRequest<CreateEmbeddingsResponse>(path, UnityWebRequest.kHttpVerbPOST, payload);
         }
+
+        /// <summary>
+        ///     Transcribes audio into the input language.
+        /// </summary>
+        /// <param name="request">See <see cref="CreateAudioTranscriptionsRequest"/></param>
+        /// <returns>See <see cref="CreateAudioResponse"/></returns>
+        public async Task<CreateAudioResponse> CreateAudioTranscription(CreateAudioTranscriptionsRequest request)
+        {
+            var path = $"{BASE_PATH}/audio/transcriptions";
+            
+            var form = new List<IMultipartFormSection>();
+            form.AddFile(request.File, "file", $"audio/{Path.GetExtension(request.File)}");
+            form.AddValue(request.Model, "model");
+            form.AddValue(request.Prompt, "prompt");
+            form.AddValue(request.ResponseFormat, "response_format");
+            form.AddValue(request.Temperature, "temperature");
+            form.AddValue(request.Language, "language");
+
+            return await DispatchRequest<CreateAudioResponse>(path, form);
+        }
+        
+        /// <summary>
+        ///     Translates audio into into English.
+        /// </summary>
+        /// <param name="request">See <see cref="CreateAudioTranslationRequest"/></param>
+        /// <returns>See <see cref="CreateAudioResponse"/></returns>
+        public async Task<CreateAudioResponse> CreateAudioTranslation(CreateAudioTranslationRequest request)
+        {
+            var path = $"{BASE_PATH}/audio/translations";
+            
+            var form = new List<IMultipartFormSection>();
+            form.AddFile(request.File, "file", $"audio/{Path.GetExtension(request.File)}");
+            form.AddValue(request.Model, "model");
+            form.AddValue(request.Prompt, "prompt");
+            form.AddValue(request.ResponseFormat, "response_format");
+            form.AddValue(request.Temperature, "temperature");
+
+            return await DispatchRequest<CreateAudioResponse>(path, form);
+        }
 
         /// <summary>
         ///     Returns a list of files that belong to the user's organization.
@@ -270,7 +315,7 @@ public async Task<OpenAIFile> CreateFile(CreateFileRequest request)
             var path = $"{BASE_PATH}/files";
 
             var form = new List<IMultipartFormSection>();
-            form.AddJsonl(request.File, "file");
+            form.AddFile(request.File, "file", "application/json");
             form.AddValue(request.Purpose, "purpose");
 
             return await DispatchRequest<OpenAIFileResponse>(path, form);

Runtime/Utils/ExtensionMethods.cs

@@ -7,34 +7,19 @@ namespace OpenAI
     public static class ExtensionMethods
     {
         /// <summary>
-        ///     Read a PNG file and add it to this form.
+        ///     Read a file and add it to this form.
         /// </summary>
         /// <param name="form">List of multipart form sections.</param>
         /// <param name="path">Path of the file to read.</param>
         /// <param name="name">Name of the form field.</param>
-        public static void AddImage(this List<IMultipartFormSection> form, string path, string name)
+        /// <param name="contentType">Content type of the file.</param>
+        public static void AddFile(this List<IMultipartFormSection> form, string path, string name, string contentType)
         {
             if (path != null)
             {
                 var data = File.ReadAllBytes(path);
                 var fileName = Path.GetFileName(path);
-                form.Add(new MultipartFormFileSection(name, data, fileName, "image/png"));
-            }
-        }
-        
-        /// <summary>
-        ///     Read a JSONL file and add it to this form.
-        /// </summary>
-        /// <param name="form">List of multipart form sections.</param>
-        /// <param name="path">Path of the file to read.</param>
-        /// <param name="name">Name of the form field.</param>
-        public static void AddJsonl(this List<IMultipartFormSection> form, string path, string name)
-        {
-            if (path != null)
-            {
-                var data = File.ReadAllBytes(path);
-                var fileName = Path.GetFileName(path);
-                form.Add(new MultipartFormFileSection(name, data, fileName, "application/json"));
+                form.Add(new MultipartFormFileSection(name, data, fileName, contentType));
             }
         }
 

Samples~/Whisper.meta

@@ -0,0 +1,184 @@
+//	Copyright (c) 2012 Calvin Rien
+//        http://the.darktable.com
+//
+//	This software is provided 'as-is', without any express or implied warranty. In
+//	no event will the authors be held liable for any damages arising from the use
+//	of this software.
+//
+//	Permission is granted to anyone to use this software for any purpose,
+//	including commercial applications, and to alter it and redistribute it freely,
+//	subject to the following restrictions:
+//
+//	1. The origin of this software must not be misrepresented; you must not claim
+//	that you wrote the original software. If you use this software in a product,
+//	an acknowledgment in the product documentation would be appreciated but is not
+//	required.
+//
+//	2. Altered source versions must be plainly marked as such, and must not be
+//	misrepresented as being the original software.
+//
+//	3. This notice may not be removed or altered from any source distribution.
+//
+//  =============================================================================
+//
+//  derived from Gregorio Zanon's script
+//  http://forum.unity3d.com/threads/119295-Writing-AudioListener.GetOutputData-to-wav-problem?p=806734&viewfull=1#post806734
+
+using System;
+using System.IO;
+using UnityEngine;
+using System.Collections.Generic;
+
+public static class SavWav {
+
+	const int HEADER_SIZE = 44;
+
+	public static bool Save(string filename, AudioClip clip) {
+		if (!filename.ToLower().EndsWith(".wav")) {
+			filename += ".wav";
+		}
+
+		var filepath = Path.Combine(Application.persistentDataPath, filename);
+
+		// Make sure directory exists if user is saving to sub dir.
+		Directory.CreateDirectory(Path.GetDirectoryName(filepath));
+
+		using (var fileStream = CreateEmpty(filepath)) {
+
+			ConvertAndWrite(fileStream, clip);
+
+			WriteHeader(fileStream, clip);
+		}
+
+		return true; // TODO: return false if there's a failure saving the file
+	}
+
+	public static AudioClip TrimSilence(AudioClip clip, float min) {
+		var samples = new float[clip.samples];
+
+		clip.GetData(samples, 0);
+
+		return TrimSilence(new List<float>(samples), min, clip.channels, clip.frequency);
+	}
+
+	public static AudioClip TrimSilence(List<float> samples, float min, int channels, int hz) {
+		return TrimSilence(samples, min, channels, hz, false, false);
+	}
+
+	public static AudioClip TrimSilence(List<float> samples, float min, int channels, int hz, bool _3D, bool stream) {
+		int i;
+
+		for (i=0; i<samples.Count; i++) {
+			if (Mathf.Abs(samples[i]) > min) {
+				break;
+			}
+		}
+
+		samples.RemoveRange(0, i);
+
+		for (i=samples.Count - 1; i>0; i--) {
+			if (Mathf.Abs(samples[i]) > min) {
+				break;
+			}
+		}
+
+		samples.RemoveRange(i, samples.Count - i);
+
+		var clip = AudioClip.Create("TempClip", samples.Count, channels, hz, _3D, stream);
+
+		clip.SetData(samples.ToArray(), 0);
+
+		return clip;
+	}
+
+	static FileStream CreateEmpty(string filepath) {
+		var fileStream = new FileStream(filepath, FileMode.Create);
+	    byte emptyByte = new byte();
+
+	    for(int i = 0; i < HEADER_SIZE; i++) //preparing the header
+	    {
+	        fileStream.WriteByte(emptyByte);
+	    }
+
+		return fileStream;
+	}
+
+	static void ConvertAndWrite(FileStream fileStream, AudioClip clip) {
+
+		var samples = new float[clip.samples];
+
+		clip.GetData(samples, 0);
+
+		Int16[] intData = new Int16[samples.Length];
+		//converting in 2 float[] steps to Int16[], //then Int16[] to Byte[]
+
+		Byte[] bytesData = new Byte[samples.Length * 2];
+		//bytesData array is twice the size of
+		//dataSource array because a float converted in Int16 is 2 bytes.
+
+		int rescaleFactor = 32767; //to convert float to Int16
+
+		for (int i = 0; i<samples.Length; i++) {
+			intData[i] = (short) (samples[i] * rescaleFactor);
+			Byte[] byteArr = new Byte[2];
+			byteArr = BitConverter.GetBytes(intData[i]);
+			byteArr.CopyTo(bytesData, i * 2);
+		}
+
+		fileStream.Write(bytesData, 0, bytesData.Length);
+	}
+
+	static void WriteHeader(FileStream fileStream, AudioClip clip) {
+
+		var hz = clip.frequency;
+		var channels = clip.channels;
+		var samples = clip.samples;
+
+		fileStream.Seek(0, SeekOrigin.Begin);
+
+		Byte[] riff = System.Text.Encoding.UTF8.GetBytes("RIFF");
+		fileStream.Write(riff, 0, 4);
+
+		Byte[] chunkSize = BitConverter.GetBytes(fileStream.Length - 8);
+		fileStream.Write(chunkSize, 0, 4);
+
+		Byte[] wave = System.Text.Encoding.UTF8.GetBytes("WAVE");
+		fileStream.Write(wave, 0, 4);
+
+		Byte[] fmt = System.Text.Encoding.UTF8.GetBytes("fmt ");
+		fileStream.Write(fmt, 0, 4);
+
+		Byte[] subChunk1 = BitConverter.GetBytes(16);
+		fileStream.Write(subChunk1, 0, 4);
+
+		UInt16 two = 2;
+		UInt16 one = 1;
+
+		Byte[] audioFormat = BitConverter.GetBytes(one);
+		fileStream.Write(audioFormat, 0, 2);
+
+		Byte[] numChannels = BitConverter.GetBytes(channels);
+		fileStream.Write(numChannels, 0, 2);
+
+		Byte[] sampleRate = BitConverter.GetBytes(hz);
+		fileStream.Write(sampleRate, 0, 4);
+
+		Byte[] byteRate = BitConverter.GetBytes(hz * channels * 2); // sampleRate * bytesPerSample*number of channels, here 44100*2*2
+		fileStream.Write(byteRate, 0, 4);
+
+		UInt16 blockAlign = (ushort) (channels * 2);
+		fileStream.Write(BitConverter.GetBytes(blockAlign), 0, 2);
+
+		UInt16 bps = 16;
+		Byte[] bitsPerSample = BitConverter.GetBytes(bps);
+		fileStream.Write(bitsPerSample, 0, 2);
+
+		Byte[] datastring = System.Text.Encoding.UTF8.GetBytes("data");
+		fileStream.Write(datastring, 0, 4);
+
+		Byte[] subChunk2 = BitConverter.GetBytes(samples * channels * 2);
+		fileStream.Write(subChunk2, 0, 4);
+
+//		fileStream.Close();
+	}
+}