Integrate audio buffer

Author: ladvoc

Runtime/Scripts/Internal/FFIClient.cs

@@ -43,6 +43,7 @@ internal sealed class FfiClient : IFFIClient
         // participant events are not allowed in the fii protocol public event ParticipantEventReceivedDelegate ParticipantEventReceived;
         public event VideoStreamEventReceivedDelegate? VideoStreamEventReceived;
         public event AudioStreamEventReceivedDelegate? AudioStreamEventReceived;
+        public event CaptureAudioFrameReceivedDelegate? CaptureAudioFrameReceived;
 
         public event PerformRpcReceivedDelegate? PerformRpcReceived;
 
@@ -275,6 +276,7 @@ static unsafe void FFICallback(UIntPtr data, UIntPtr size)
                         Instance.AudioStreamEventReceived?.Invoke(r.AudioStreamEvent!);
                         break;
                     case FfiEvent.MessageOneofCase.CaptureAudioFrame:
+                         Instance.CaptureAudioFrameReceived?.Invoke(r.CaptureAudioFrame!);
                         break;
                     case FfiEvent.MessageOneofCase.PerformRpc:
                         Instance.PerformRpcReceived?.Invoke(r.PerformRpc!);

Runtime/Scripts/Internal/FFIClients/FFIEvents.cs

@@ -44,6 +44,8 @@ namespace LiveKit.Internal
 
     internal delegate void SendTextReceivedDelegate(StreamSendTextCallback e);
 
+    internal delegate void CaptureAudioFrameReceivedDelegate(CaptureAudioFrameCallback e);
+
     // Events
     internal delegate void RoomEventReceivedDelegate(RoomEvent e);
 

Runtime/Scripts/RtcAudioSource.cs

@@ -18,12 +18,12 @@ public enum RtcAudioSourceType
     public abstract class RtcAudioSource : IRtcSource
     {
         public abstract event Action<float[], int, int> AudioRead;
-        public virtual IEnumerator Prepare(float timeout = 0) { yield break;  }
+        public virtual IEnumerator Prepare(float timeout = 0) { yield break; }
         public abstract void Play();
 
 #if UNITY_IOS
         // iOS microphone sample rate is 24k,
-        // please make sure when you using 
+        // please make sure when you using
         // sourceType is AudioSourceMicrophone
         public static uint DefaultMirophoneSampleRate = 24000;
 
@@ -43,7 +43,7 @@ public abstract class RtcAudioSource : IRtcSource
 
         // Possibly used on the AudioThread
         private Thread _readAudioThread;
-        private ThreadSafeQueue<AudioFrame> _frameQueue = new ThreadSafeQueue<AudioFrame>();
+        private AudioBuffer _captureBuffer = new AudioBuffer();
 
         private bool _muted = false;
         public override bool Muted => _muted;
@@ -85,7 +85,6 @@ public void Start()
             Stop();
             _readAudioThread = new Thread(Update);
             _readAudioThread.Start();
-
             AudioRead += OnAudioRead;
             Play();
         }
@@ -101,78 +100,55 @@ private void Update()
             while (true)
             {
                 Thread.Sleep(Constants.TASK_DELAY);
-                ReadAudio();
+                var frame = _captureBuffer.ReadDuration(10); // 10ms
+                if (_muted || frame == null) continue;
+                Capture(frame);
             }
         }
 
         private void OnAudioRead(float[] data, int channels, int sampleRate)
         {
-            var samplesPerChannel = data.Length / channels;
-            var frame = new AudioFrame((uint)sampleRate, (uint)channels, (uint)samplesPerChannel);
-
-            static short FloatToS16(float v)
-            {
-                v *= 32768f;
-                v = Math.Min(v, 32767f);
-                v = Math.Max(v, -32768f);
-                return (short)(v + Math.Sign(v) * 0.5f);
-            }
-            unsafe
+            _captureBuffer.Write(data, (uint)channels, (uint)sampleRate);
+            if (_sourceType == RtcAudioSourceType.AudioSourceMicrophone)
             {
-                var frameData = new Span<short>(frame.Data.ToPointer(), frame.Length / sizeof(short));
-                for (int i = 0; i < data.Length; i++)
-                {
-                    frameData[i] = FloatToS16(data[i]);
-                }
-                if (_sourceType == RtcAudioSourceType.AudioSourceMicrophone)
-                {
-                   // Don't play the audio locally, to avoid echo.
-                    Array.Clear(data, 0, data.Length);
-                }
+                // Don't play the audio locally, to avoid echo.
+                Array.Clear(data, 0, data.Length);
             }
-            _frameQueue.Enqueue(frame);
         }
 
-        private void ReadAudio()
+        private void Capture(AudioFrame frame)
         {
-            while (_frameQueue.Count > 0)
+            using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
+            using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
+
+            var pushFrame = request.request;
+            pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
+
+            pushFrame.Buffer = audioFrameBufferInfo;
+            pushFrame.Buffer.DataPtr = (ulong)frame.Data;
+            pushFrame.Buffer.NumChannels = frame.NumChannels;
+            pushFrame.Buffer.SampleRate = frame.SampleRate;
+            pushFrame.Buffer.SamplesPerChannel = frame.SamplesPerChannel;
+
+            using var response = request.Send();
+            FfiResponse res = response;
+
+            // Frame needs to stay alive until receiving the async callback.
+            var asyncId = res.CaptureAudioFrame.AsyncId;
+            void Callback(CaptureAudioFrameCallback callback)
             {
-                try
-                {
-                    AudioFrame frame = _frameQueue.Dequeue();
-
-                    if(_muted)
-                    {
-                        continue;
-                    }
-                    unsafe
-                    {
-                        using var request = FFIBridge.Instance.NewRequest<CaptureAudioFrameRequest>();
-                        using var audioFrameBufferInfo = request.TempResource<AudioFrameBufferInfo>();
-
-                        var pushFrame = request.request;
-                        pushFrame.SourceHandle = (ulong)Handle.DangerousGetHandle();
-
-                        pushFrame.Buffer = audioFrameBufferInfo;
-                        pushFrame.Buffer.DataPtr = (ulong)frame.Data;
-                        pushFrame.Buffer.NumChannels = frame.NumChannels;
-                        pushFrame.Buffer.SampleRate = frame.SampleRate;
-                        pushFrame.Buffer.SamplesPerChannel = frame.SamplesPerChannel;
-
-                        using var response = request.Send();
-                    }
-                }
-                catch (Exception e)
-                {
-                    Utils.Error("Audio Framedata error: " + e.Message);
-                }
+                if (callback.AsyncId != asyncId) return;
+                if (callback.HasError)
+                    Utils.Error($"Audio capture failed: {callback.Error}");
+                frame.Dispose();
+                FfiClient.Instance.CaptureAudioFrameReceived -= Callback;
             }
+            FfiClient.Instance.CaptureAudioFrameReceived += Callback;
         }
 
         public override void SetMute(bool muted)
         {
             _muted = muted;
         }
-
     }
-}
+}