feat: support rendering callback for rtc video source. (#43)

cloudwebrtc · web-flow · commit da21284f2a1d · 2024-07-22T18:21:04.000+08:00
* feat: support rendering callback for rtc video source.

* update.

* bump version.

* update.

* texture dispose.
diff --git a/Runtime/Scripts/CameraVideoSource.cs b/Runtime/Scripts/CameraVideoSource.cs
@@ -52,12 +52,12 @@ private void ClearRenderTexture()
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
-
+            var textureChanged = false;
             try
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -69,6 +69,7 @@ protected override void ReadBuffer()
                     _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
                     Camera.targetTexture = _dest as RenderTexture;
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
+                    textureChanged = true;
                 }
                 ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
                 AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
@@ -77,7 +78,7 @@ protected override void ReadBuffer()
             {
                 Utils.Error(e);
             }
-        
+            return textureChanged;
         }
     }
 }
diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs
@@ -26,6 +26,10 @@ public enum VideoStreamSource
         public abstract int GetWidth();
         public abstract int GetHeight();
 
+        public delegate void TextureReceiveDelegate(Texture2D tex2d);
+        /// Called when we receive a new texture (first texture or the resolution changed)
+        public event TextureReceiveDelegate TextureReceived;
+
         protected Texture _dest;
         protected NativeArray<byte> _data;
         protected VideoStreamSource _sourceType;
@@ -35,6 +39,7 @@ public enum VideoStreamSource
         protected bool _requestPending = false;
         protected bool isDisposed = true;
         protected bool _playing = false;
+        private Texture2D _texture2D = null;
 
         internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
         {
@@ -121,12 +126,41 @@ public virtual void Stop()
             _playing = false; 
         }
 
+        private void LoadToTexture2D(Texture2D tex, RenderTexture rTex)
+        {
+            var old_rt = RenderTexture.active;
+            RenderTexture.active = rTex;
+
+            tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);
+            tex.Apply();
+
+            RenderTexture.active = old_rt;
+        }
+
         public IEnumerator Update()
         {
             while (_playing)
             {
                 yield return null;
-                ReadBuffer();
+                var textureChanged = ReadBuffer();
+
+                if(textureChanged)
+                {
+                    if (_texture2D == null)
+                    {
+                        _texture2D = new Texture2D(_dest.width, _dest.height, TextureFormat.RGB24, false);
+                    } else
+                    {
+                        _texture2D.Reinitialize(_dest.width, _dest.height);
+                    }
+                    TextureReceived?.Invoke(_texture2D);
+                }
+
+                if(TextureReceived.GetInvocationList().Length > 0)
+                {
+                    LoadToTexture2D(_texture2D, _dest as RenderTexture);
+                }
+
                 SendFrame();
             }
 
@@ -136,13 +170,14 @@ public IEnumerator Update()
         public virtual void Dispose()
         {
             if (!isDisposed)
-            { 
-                _data.Dispose();
+            {
+                if (_data != null) _data.Dispose();
+                if (_texture2D != null) UnityEngine.Object.Destroy(_texture2D);
                 isDisposed = true;
             }
         }
 
-        protected abstract void ReadBuffer();
+        protected abstract bool ReadBuffer();
 
         protected virtual bool SendFrame()
         {
diff --git a/Runtime/Scripts/ScreenVideoSource.cs b/Runtime/Scripts/ScreenVideoSource.cs
@@ -49,11 +49,12 @@ private void ClearRenderTexture()
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
+            var textureChanged = false;
             try
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -64,6 +65,7 @@ protected override void ReadBuffer()
                     _bufferType = GetVideoBufferType(_textureFormat);
                     _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
+                    textureChanged = true;
                 }
                 ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
                 AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
@@ -72,6 +74,7 @@ protected override void ReadBuffer()
             {
                 Utils.Error(e);
             }
+            return textureChanged;
         }
 
         protected override bool SendFrame()
diff --git a/Runtime/Scripts/TextureVideoSource.cs b/Runtime/Scripts/TextureVideoSource.cs
@@ -34,11 +34,12 @@ public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBuf
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
+            var textureChanged = false;
             if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -48,17 +49,24 @@ protected override void ReadBuffer()
                     _bufferType = GetVideoBufferType(_textureFormat);
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
                     _dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
+                    textureChanged = true;
                 }
                 Graphics.CopyTexture(Texture, _dest);
             }
             else
             {
+                if(_dest == null || _dest != Texture)
+                {
+                    textureChanged = true;
+                }
+
                 _dest = Texture;
                 _textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);
                 _bufferType = GetVideoBufferType(_textureFormat);
             }
             
             AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
+            return textureChanged;
         }
     }
 }
diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs
@@ -71,7 +71,7 @@ private void Dispose(bool disposing)
             {
                 if (disposing)
                     VideoBuffer?.Dispose();
-
+                if (Texture != null) UnityEngine.Object.Destroy(Texture);
                 _disposed = true;
             }
         }
@@ -106,7 +106,7 @@ public IEnumerator Update()
                 var textureChanged = false;
                 if (Texture == null || Texture.width != rWidth || Texture.height != rHeight)
                 {
-                   if (Texture != null) UnityEngine.Object.Destroy(Texture);
+                    if (Texture != null) UnityEngine.Object.Destroy(Texture);
                     Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false);
                     Texture.ignoreMipmapLimit = false;
                     textureChanged = true;
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "io.livekit.livekit-sdk",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "displayName": "LiveKit SDK",
   "description": "LiveKit",
   "unity": "2021.3",

Original file line number	Diff line number	Diff line change
`@@ -52,12 +52,12 @@ private void ClearRenderTexture()`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`// Read the texture data into a native array asynchronously`
`55`		`- protected override void ReadBuffer()`
	`55`	`+ protected override bool ReadBuffer()`
`56`	`56`	`{`
`57`	`57`	`if (_reading)`
`58`		`- return;`
	`58`	`+ return false;`
`59`	`59`	`_reading = true;`
`60`		`-`
	`60`	`+ var textureChanged = false;`
`61`	`61`	`try`
`62`	`62`	`{`
`63`	`63`	`if (_dest == null \|\| _dest.width != GetWidth() \|\| _dest.height != GetHeight())`
`@@ -69,6 +69,7 @@ protected override void ReadBuffer()`
`69`	`69`	`_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);`
`70`	`70`	`Camera.targetTexture = _dest as RenderTexture;`
`71`	`71`	`_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);`
	`72`	`+ textureChanged = true;`
`72`	`73`	`}`
`73`	`74`	`ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);`
`74`	`75`	`AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);`
`@@ -77,7 +78,7 @@ protected override void ReadBuffer()`
`77`	`78`	`{`
`78`	`79`	`Utils.Error(e);`
`79`	`80`	`}`
`80`		`-`
	`81`	`+ return textureChanged;`
`81`	`82`	`}`
`82`	`83`	`}`
`83`	`84`	`}`
Original file line number	Diff line number	Diff line change
`@@ -49,11 +49,12 @@ private void ClearRenderTexture()`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`// Read the texture data into a native array asynchronously`
`52`		`- protected override void ReadBuffer()`
	`52`	`+ protected override bool ReadBuffer()`
`53`	`53`	`{`
`54`	`54`	`if (_reading)`
`55`		`- return;`
	`55`	`+ return false;`
`56`	`56`	`_reading = true;`
	`57`	`+ var textureChanged = false;`
`57`	`58`	`try`
`58`	`59`	`{`
`59`	`60`	`if (_dest == null \|\| _dest.width != GetWidth() \|\| _dest.height != GetHeight())`
`@@ -64,6 +65,7 @@ protected override void ReadBuffer()`
`64`	`65`	`_bufferType = GetVideoBufferType(_textureFormat);`
`65`	`66`	`_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);`
`66`	`67`	`_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);`
	`68`	`+ textureChanged = true;`
`67`	`69`	`}`
`68`	`70`	`ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);`
`69`	`71`	`AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);`
`@@ -72,6 +74,7 @@ protected override void ReadBuffer()`
`72`	`74`	`{`
`73`	`75`	`Utils.Error(e);`
`74`	`76`	`}`
	`77`	`+ return textureChanged;`
`75`	`78`	`}`
`76`	`79`
`77`	`80`	`protected override bool SendFrame()`
Original file line number	Diff line number	Diff line change
`@@ -34,11 +34,12 @@ public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBuf`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`// Read the texture data into a native array asynchronously`
`37`		`- protected override void ReadBuffer()`
	`37`	`+ protected override bool ReadBuffer()`
`38`	`38`	`{`
`39`	`39`	`if (_reading)`
`40`		`- return;`
	`40`	`+ return false;`
`41`	`41`	`_reading = true;`
	`42`	`+ var textureChanged = false;`
`42`	`43`	`if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))`
`43`	`44`	`{`
`44`	`45`	`if (_dest == null \|\| _dest.width != GetWidth() \|\| _dest.height != GetHeight())`
`@@ -48,17 +49,24 @@ protected override void ReadBuffer()`
`48`	`49`	`_bufferType = GetVideoBufferType(_textureFormat);`
`49`	`50`	`_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);`
`50`	`51`	`_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);`
	`52`	`+ textureChanged = true;`
`51`	`53`	`}`
`52`	`54`	`Graphics.CopyTexture(Texture, _dest);`
`53`	`55`	`}`
`54`	`56`	`else`
`55`	`57`	`{`
	`58`	`+ if(_dest == null \|\| _dest != Texture)`
	`59`	`+ {`
	`60`	`+ textureChanged = true;`
	`61`	`+ }`
	`62`	`+`
`56`	`63`	`_dest = Texture;`
`57`	`64`	`_textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);`
`58`	`65`	`_bufferType = GetVideoBufferType(_textureFormat);`
`59`	`66`	`}`
`60`	`67`
`61`	`68`	`AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);`
	`69`	`+ return textureChanged;`
`62`	`70`	`}`
`63`	`71`	`}`
`64`	`72`	`}`
Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ private void Dispose(bool disposing)`
`71`	`71`	`{`
`72`	`72`	`if (disposing)`
`73`	`73`	`VideoBuffer?.Dispose();`
`74`		`-`
	`74`	`+ if (Texture != null) UnityEngine.Object.Destroy(Texture);`
`75`	`75`	`_disposed = true;`
`76`	`76`	`}`
`77`	`77`	`}`
`@@ -106,7 +106,7 @@ public IEnumerator Update()`
`106`	`106`	`var textureChanged = false;`
`107`	`107`	`if (Texture == null \|\| Texture.width != rWidth \|\| Texture.height != rHeight)`
`108`	`108`	`{`
`109`		`- if (Texture != null) UnityEngine.Object.Destroy(Texture);`
	`109`	`+ if (Texture != null) UnityEngine.Object.Destroy(Texture);`
`110`	`110`	`Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false);`
`111`	`111`	`Texture.ignoreMipmapLimit = false;`
`112`	`112`	`textureChanged = true;`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "io.livekit.livekit-sdk",`
`3`		`- "version": "1.0.0",`
	`3`	`+ "version": "1.0.1",`
`4`	`4`	`"displayName": "LiveKit SDK",`
`5`	`5`	`"description": "LiveKit",`
`6`	`6`	`"unity": "2021.3",`