Whisper Endpoint ByteArray Request (#31)

* feat: file data type

* feat: ext method for adding file directly from bytes

* feat: if file path does not exists use file data

* feat: savewav file updated to use memory stream

* feat: whisper sample uses file bytes. no file saved locally

Author: srcnalt

Runtime/DataTypes.cs

@@ -120,9 +120,16 @@ public struct ChatMessage
 
     #region Audio Transcriptions Data Types
 
+    public struct FileData
+    {
+        public byte[] Data;
+        public string Name;
+    }
+
     public class CreateAudioRequestBase
     {
         public string File { get; set; }
+        public FileData FileData { get; set; }
         public string Model { get; set; }
         public string Prompt { get; set; }
         public string ResponseFormat { get; set; } = AudioResponseFormat.Json;

Runtime/OpenAIApi.cs

@@ -264,7 +264,14 @@ public async Task<CreateAudioResponse> CreateAudioTranscription(CreateAudioTrans
             var path = $"{BASE_PATH}/audio/transcriptions";
 
             var form = new List<IMultipartFormSection>();
-            form.AddFile(request.File, "file", $"audio/{Path.GetExtension(request.File)}");
+            if (string.IsNullOrEmpty(request.File))
+            {
+                form.AddData(request.FileData, "file", $"audio/{Path.GetExtension(request.File)}");
+            }
+            else
+            {
+                form.AddFile(request.File, "file", $"audio/{Path.GetExtension(request.File)}");
+            }
             form.AddValue(request.Model, "model");
             form.AddValue(request.Prompt, "prompt");
             form.AddValue(request.ResponseFormat, "response_format");

Runtime/Utils/ExtensionMethods.cs

@@ -23,6 +23,22 @@ public static void AddFile(this List<IMultipartFormSection> form, string path, s
             }
         }
 
+        /// <summary>
+        ///     Read a file and add it to this form.
+        /// </summary>
+        /// <param name="form">List of multipart form sections.</param>
+        /// <param name="data">Byte array data of the file to attach.</param>
+        /// <param name="name">Name of the form field.</param>
+        /// <param name="contentType">Content type of the file.</param>
+        public static void AddData(this List<IMultipartFormSection> form, FileData data, string name, string contentType)
+        {
+            if (data.Data != null)
+            {
+                var fileName = Path.GetFileName(data.Name);
+                form.Add(new MultipartFormFileSection(name, data.Data, fileName, contentType));
+            }
+        }
+        
         /// <summary>
         ///     Add a primitive value to the form.
         /// </summary>

Samples~/Whisper/SavWav.cs renamed to Samples~/Whisper/SaveWav.cs

@@ -29,11 +29,11 @@
 using UnityEngine;
 using System.Collections.Generic;
 
-public static class SavWav {
+public static class SaveWav {
 
 	const int HEADER_SIZE = 44;
 
-	public static bool Save(string filename, AudioClip clip) {
+	public static byte[] Save(string filename, AudioClip clip) {
 		if (!filename.ToLower().EndsWith(".wav")) {
 			filename += ".wav";
 		}
@@ -43,14 +43,12 @@ public static bool Save(string filename, AudioClip clip) {
 		// Make sure directory exists if user is saving to sub dir.
 		Directory.CreateDirectory(Path.GetDirectoryName(filepath));
 
-		using (var fileStream = CreateEmpty(filepath)) {
-
-			ConvertAndWrite(fileStream, clip);
-
-			WriteHeader(fileStream, clip);
+		using (var memoryStream = CreateEmpty(filepath))
+		{
+			ConvertAndWrite(memoryStream, clip);
+			WriteHeader(memoryStream, clip);
+			return memoryStream.GetBuffer();
 		}
-
-		return true; // TODO: return false if there's a failure saving the file
 	}
 
 	public static AudioClip TrimSilence(AudioClip clip, float min) {
@@ -91,19 +89,19 @@ public static AudioClip TrimSilence(List<float> samples, float min, int channels
 		return clip;
 	}
 
-	static FileStream CreateEmpty(string filepath) {
-		var fileStream = new FileStream(filepath, FileMode.Create);
-	    byte emptyByte = new byte();
+	static MemoryStream CreateEmpty(string filepath) {
+		var memoryStream = new MemoryStream();
+    byte emptyByte = new byte();
 
-	    for(int i = 0; i < HEADER_SIZE; i++) //preparing the header
-	    {
-	        fileStream.WriteByte(emptyByte);
-	    }
+    for(int i = 0; i < HEADER_SIZE; i++) //preparing the header
+    {
+	    memoryStream.WriteByte(emptyByte);
+    }
 
-		return fileStream;
+		return memoryStream;
 	}
 
-	static void ConvertAndWrite(FileStream fileStream, AudioClip clip) {
+	static void ConvertAndWrite(MemoryStream memoryStream, AudioClip clip) {
 
 		var samples = new float[clip.samples];
 
@@ -125,60 +123,60 @@ static void ConvertAndWrite(FileStream fileStream, AudioClip clip) {
 			byteArr.CopyTo(bytesData, i * 2);
 		}
 
-		fileStream.Write(bytesData, 0, bytesData.Length);
+		memoryStream.Write(bytesData, 0, bytesData.Length);
 	}
 
-	static void WriteHeader(FileStream fileStream, AudioClip clip) {
+	static void WriteHeader(MemoryStream memoryStream, AudioClip clip) {
 
 		var hz = clip.frequency;
 		var channels = clip.channels;
 		var samples = clip.samples;
 
-		fileStream.Seek(0, SeekOrigin.Begin);
+		memoryStream.Seek(0, SeekOrigin.Begin);
 
 		Byte[] riff = System.Text.Encoding.UTF8.GetBytes("RIFF");
-		fileStream.Write(riff, 0, 4);
+		memoryStream.Write(riff, 0, 4);
 
-		Byte[] chunkSize = BitConverter.GetBytes(fileStream.Length - 8);
-		fileStream.Write(chunkSize, 0, 4);
+		Byte[] chunkSize = BitConverter.GetBytes(memoryStream.Length - 8);
+		memoryStream.Write(chunkSize, 0, 4);
 
 		Byte[] wave = System.Text.Encoding.UTF8.GetBytes("WAVE");
-		fileStream.Write(wave, 0, 4);
+		memoryStream.Write(wave, 0, 4);
 
 		Byte[] fmt = System.Text.Encoding.UTF8.GetBytes("fmt ");
-		fileStream.Write(fmt, 0, 4);
+		memoryStream.Write(fmt, 0, 4);
 
 		Byte[] subChunk1 = BitConverter.GetBytes(16);
-		fileStream.Write(subChunk1, 0, 4);
+		memoryStream.Write(subChunk1, 0, 4);
 
 		UInt16 two = 2;
 		UInt16 one = 1;
 
 		Byte[] audioFormat = BitConverter.GetBytes(one);
-		fileStream.Write(audioFormat, 0, 2);
+		memoryStream.Write(audioFormat, 0, 2);
 
 		Byte[] numChannels = BitConverter.GetBytes(channels);
-		fileStream.Write(numChannels, 0, 2);
+		memoryStream.Write(numChannels, 0, 2);
 
 		Byte[] sampleRate = BitConverter.GetBytes(hz);
-		fileStream.Write(sampleRate, 0, 4);
+		memoryStream.Write(sampleRate, 0, 4);
 
 		Byte[] byteRate = BitConverter.GetBytes(hz * channels * 2); // sampleRate * bytesPerSample*number of channels, here 44100*2*2
-		fileStream.Write(byteRate, 0, 4);
+		memoryStream.Write(byteRate, 0, 4);
 
 		UInt16 blockAlign = (ushort) (channels * 2);
-		fileStream.Write(BitConverter.GetBytes(blockAlign), 0, 2);
+		memoryStream.Write(BitConverter.GetBytes(blockAlign), 0, 2);
 
 		UInt16 bps = 16;
 		Byte[] bitsPerSample = BitConverter.GetBytes(bps);
-		fileStream.Write(bitsPerSample, 0, 2);
+		memoryStream.Write(bitsPerSample, 0, 2);
 
 		Byte[] datastring = System.Text.Encoding.UTF8.GetBytes("data");
-		fileStream.Write(datastring, 0, 4);
+		memoryStream.Write(datastring, 0, 4);
 
 		Byte[] subChunk2 = BitConverter.GetBytes(samples * channels * 2);
-		fileStream.Write(subChunk2, 0, 4);
+		memoryStream.Write(subChunk2, 0, 4);
 
 //		fileStream.Close();
 	}
-}
+}

Samples~/Whisper/SavWav.cs.meta renamed to Samples~/Whisper/SaveWav.cs.meta

@@ -42,11 +42,12 @@ private async void EndRecording()
 
             isRecording = false;
             Microphone.End(null);
-            SavWav.Save(fileName, clip);
+            byte[] data = SaveWav.Save(fileName, clip);
 
             var req = new CreateAudioTranscriptionsRequest
             {
-                File = Application.persistentDataPath + "/" + fileName,
+                FileData = new FileData() {Data = data, Name = "audio.wav"},
+                // File = Application.persistentDataPath + "/" + fileName,
                 Model = "whisper-1",
                 Language = "en"
             };