Integrate apm

Author: ladvoc

Runtime/Scripts/ApmReverseStream.cs

@@ -0,0 +1,63 @@
+using System.Threading;
+using LiveKit.Internal;
+using UnityEngine;
+
+namespace LiveKit
+{
+    /// <summary>
+    /// Captures and processes the reverse audio stream using an <see cref="AudioProcessingModule"/>.
+    /// </summary>
+    /// <remarks>
+    /// The reverse stream is captured from the scene's audio listener.
+    /// </remarks>
+    internal class ApmReverseStream
+    {
+        private readonly AudioBuffer _captureBuffer = new AudioBuffer();
+        private readonly AudioProcessingModule _apm; // APM is thread safe
+        private Thread _thread;
+        private AudioFilter _audioFilter;
+
+        internal ApmReverseStream(AudioProcessingModule apm)
+        {
+            _apm = apm;
+        }
+
+        internal void Start()
+        {
+            var audioListener = GameObject.FindObjectOfType<AudioListener>();
+            if (audioListener == null)
+            {
+                Utils.Error("AudioListener not found in scene");
+                return;
+            }
+            _audioFilter = audioListener.gameObject.AddComponent<AudioFilter>();
+            _audioFilter.AudioRead += OnAudioRead;
+
+            _thread = new Thread(ProcessReverseStream);
+            _thread.Start();
+        }
+
+        internal void Stop()
+        {
+            _thread?.Abort();
+            if (_audioFilter != null)
+                Object.Destroy(_audioFilter);
+        }
+
+        private void ProcessReverseStream()
+        {
+            while (true)
+            {
+                Thread.Sleep(Constants.TASK_DELAY);
+                using var frame = _captureBuffer.ReadDuration(AudioProcessingModule.FRAME_DURATION_MS);
+                if (frame == null) continue;
+                _apm.ProcessReverseStream(frame);
+            }
+        }
+
+        private void OnAudioRead(float[] data, int channels, int sampleRate)
+        {
+            _captureBuffer.Write(data, (uint)channels, (uint)sampleRate);
+        }
+    }
+}

Runtime/Scripts/ApmReverseStream.cs.meta

@@ -126,5 +126,10 @@ public void SetStreamDelayMs(int delayMs)
                 throw new Exception(res.ApmSetStreamDelay.Error);
             }
         }
+
+        /// <summary>
+        /// The required duration for audio frames being processed.
+        /// </summary>
+        public const uint FRAME_DURATION_MS = 10;
     }
 }

Runtime/Scripts/AudioProcessingModule.cs

@@ -44,30 +44,26 @@ public abstract class RtcAudioSource : IRtcSource
         // Possibly used on the AudioThread
         private Thread _readAudioThread;
         private AudioBuffer _captureBuffer = new AudioBuffer();
+        private readonly AudioProcessingModule _apm;
+        private readonly ApmReverseStream _apmReverseStream;
 
         private bool _muted = false;
         public override bool Muted => _muted;
 
         protected RtcAudioSource(int channels = 2, RtcAudioSourceType audioSourceType = RtcAudioSourceType.AudioSourceCustom)
         {
+            var isMicrophone = audioSourceType == RtcAudioSourceType.AudioSourceMicrophone;
             _sourceType = audioSourceType;
+            _apm = new AudioProcessingModule(isMicrophone, true, true, true);
+            if (isMicrophone)
+                _apmReverseStream = new ApmReverseStream(_apm);
 
             using var request = FFIBridge.Instance.NewRequest<NewAudioSourceRequest>();
             var newAudioSource = request.request;
             newAudioSource.Type = AudioSourceType.AudioSourceNative;
             newAudioSource.NumChannels = (uint)channels;
-            if(_sourceType == RtcAudioSourceType.AudioSourceMicrophone)
-            {
-                newAudioSource.SampleRate = DefaultMirophoneSampleRate;
-            }
-            else
-            {
-                newAudioSource.SampleRate = DefaultSampleRate;
-            }
-            newAudioSource.Options = request.TempResource<AudioSourceOptions>();
-            newAudioSource.Options.EchoCancellation = true;
-            newAudioSource.Options.AutoGainControl = true;
-            newAudioSource.Options.NoiseSuppression = true;
+            newAudioSource.SampleRate = isMicrophone ? DefaultMirophoneSampleRate : DefaultSampleRate;
+
             using var response = request.Send();
             FfiResponse res = response;
             _info = res.NewAudioSource.Source.Info;
@@ -85,13 +81,15 @@ public void Start()
             Stop();
             _readAudioThread = new Thread(Update);
             _readAudioThread.Start();
+            _apmReverseStream?.Start();
             AudioRead += OnAudioRead;
             Play();
         }
 
         public virtual void Stop()
         {
             _readAudioThread?.Abort();
+            _apmReverseStream?.Stop();
             AudioRead -= OnAudioRead;
         }
 
@@ -100,8 +98,17 @@ private void Update()
             while (true)
             {
                 Thread.Sleep(Constants.TASK_DELAY);
-                var frame = _captureBuffer.ReadDuration(10); // 10ms
+                var frame = _captureBuffer.ReadDuration(AudioProcessingModule.FRAME_DURATION_MS);
                 if (_muted || frame == null) continue;
+
+                if (_apmReverseStream != null)
+                {
+                    // TODO: calculate stream delay
+                    var delayMs = 0;
+                    _apm.SetStreamDelayMs(delayMs);
+                }
+                _apm.ProcessStream(frame);
+
                 Capture(frame);
             }
         }