Skip to content

Commit 47692c7

Browse files
cloudwebrtccloudwebrtc
authored
fix: Optimize VideoSource capture and add WebCameraSource. (#59)
1 parent 9f9bfdf commit 47692c7

File tree

6 files changed

+163
-76
lines changed

6 files changed

+163
-76
lines changed

‎Runtime/Scripts/CameraVideoSource.cs

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,14 @@
88

99
namespace LiveKit
1010
{
11+
12+
// VideoSource for Unity Camera
1113
public class CameraVideoSource : RtcVideoSource
1214
{
1315
private TextureFormat _textureFormat;
1416

17+
private RenderTexture _renderTexture;
18+
1519
public Camera Camera { get; }
1620

1721
public override int GetWidth()
@@ -24,6 +28,11 @@ public override int GetHeight()
2428
return Camera.pixelHeight;
2529
}
2630

31+
protected override VideoRotation GetVideoRotation()
32+
{
33+
return VideoRotation._0;
34+
}
35+
2736
public CameraVideoSource(Camera camera, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
2837
{
2938
Camera = camera;
@@ -44,9 +53,9 @@ public override void Stop()
4453

4554
private void ClearRenderTexture()
4655
{
47-
if (_dest)
56+
if (_renderTexture)
4857
{
49-
var renderText = _dest as RenderTexture;
58+
var renderText = _renderTexture as RenderTexture;
5059
renderText.Release(); // can only be done on main thread
5160
}
5261
}
@@ -60,19 +69,35 @@ protected override bool ReadBuffer()
6069
var textureChanged = false;
6170
try
6271
{
63-
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
72+
if (_renderTexture == null || _renderTexture.width != GetWidth() || _renderTexture.height != GetHeight())
6473
{
6574
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
6675
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
6776
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
6877
_bufferType = GetVideoBufferType(_textureFormat);
69-
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
70-
Camera.targetTexture = _dest as RenderTexture;
78+
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
79+
Camera.targetTexture = _renderTexture as RenderTexture;
7180
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
81+
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
7282
textureChanged = true;
7383
}
74-
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
75-
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
84+
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);
85+
86+
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
87+
// Flip the texture for OSX
88+
Graphics.CopyTexture(_renderTexture, _dest);
89+
var pixels = _dest.GetPixels();
90+
var flippedPixels = new Color[pixels.Length];
91+
for (int i = 0; i < _dest.height; i++)
92+
{
93+
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
94+
}
95+
_dest.SetPixels(flippedPixels);
96+
#else
97+
Graphics.CopyTexture(_renderTexture, _dest);
98+
#endif
99+
100+
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
76101
}
77102
catch (Exception e)
78103
{

‎Runtime/Scripts/RtcVideoSource.cs

Lines changed: 6 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ public enum VideoStreamSource
2626
public abstract int GetWidth();
2727
public abstract int GetHeight();
2828

29+
protected abstract VideoRotation GetVideoRotation();
30+
2931
public delegate void TextureReceiveDelegate(Texture2D tex2d);
3032
/// Called when we receive a new texture (first texture or the resolution changed)
3133
public event TextureReceiveDelegate TextureReceived;
3234

33-
protected Texture _dest;
35+
protected Texture2D _dest;
3436
protected NativeArray<byte> _data;
3537
protected VideoStreamSource _sourceType;
3638
protected VideoBufferType _bufferType;
@@ -39,7 +41,6 @@ public enum VideoStreamSource
3941
protected bool _requestPending = false;
4042
protected bool isDisposed = true;
4143
protected bool _playing = false;
42-
private Texture2D _texture2D = null;
4344
private bool _muted = false;
4445
public override bool Muted => _muted;
4546

@@ -128,31 +129,6 @@ public virtual void Stop()
128129
_playing = false;
129130
}
130131

131-
private void LoadToTexture2D(Texture2D tex, RenderTexture rTex)
132-
{
133-
if (tex == null || rTex == null)
134-
{
135-
return;
136-
}
137-
138-
var old_rt = RenderTexture.active;
139-
RenderTexture.active = rTex;
140-
tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);
141-
142-
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
143-
// Flip the texture for OSX
144-
var pixels = tex.GetPixels();
145-
var flippedPixels = new Color[pixels.Length];
146-
for (int i = 0; i < rTex.height; i++)
147-
{
148-
Array.Copy(pixels, i * rTex.width, flippedPixels, (rTex.height - i - 1) * rTex.width, rTex.width);
149-
}
150-
tex.SetPixels(flippedPixels);
151-
#endif
152-
tex.Apply();
153-
RenderTexture.active = old_rt;
154-
}
155-
156132
public IEnumerator Update()
157133
{
158134
while (_playing)
@@ -162,20 +138,9 @@ public IEnumerator Update()
162138

163139
if(textureChanged)
164140
{
165-
if (_texture2D == null)
166-
{
167-
_texture2D = new Texture2D(_dest.width, _dest.height, TextureFormat.RGB24, false);
168-
} else
169-
{
170-
_texture2D.Reinitialize(_dest.width, _dest.height);
171-
}
172-
TextureReceived?.Invoke(_texture2D);
141+
TextureReceived?.Invoke(_dest);
173142
}
174143

175-
if(TextureReceived != null && TextureReceived.GetInvocationList().Length > 0)
176-
{
177-
LoadToTexture2D(_texture2D, _dest as RenderTexture);
178-
}
179144
if(_muted)
180145
{
181146
continue;
@@ -195,7 +160,7 @@ public virtual void Dispose()
195160
{
196161
if (!isDisposed)
197162
{
198-
if (_texture2D != null) UnityEngine.Object.Destroy(_texture2D);
163+
if (_dest != null) UnityEngine.Object.Destroy(_dest);
199164
isDisposed = true;
200165
}
201166
}
@@ -222,7 +187,7 @@ protected virtual bool SendFrame()
222187
using var request = FFIBridge.Instance.NewRequest<CaptureVideoFrameRequest>();
223188
var capture = request.request;
224189
capture.SourceHandle = (ulong)Handle.DangerousGetHandle();
225-
capture.Rotation = VideoRotation._0;
190+
capture.Rotation = GetVideoRotation();
226191
capture.TimestampUs = DateTimeOffset.Now.ToUnixTimeMilliseconds();
227192
capture.Buffer = buffer;
228193
using var response = request.Send();

‎Runtime/Scripts/ScreenVideoSource.cs

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace LiveKit
1111
public class ScreenVideoSource : RtcVideoSource
1212
{
1313
private TextureFormat _textureFormat;
14+
private RenderTexture _renderTexture;
1415

1516
public override int GetWidth()
1617
{
@@ -22,6 +23,11 @@ public override int GetHeight()
2223
return Screen.height;
2324
}
2425

26+
protected override VideoRotation GetVideoRotation()
27+
{
28+
return VideoRotation._0;
29+
}
30+
2531
public ScreenVideoSource(VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
2632
{
2733
base.Init();
@@ -41,9 +47,9 @@ public override void Stop()
4147

4248
private void ClearRenderTexture()
4349
{
44-
if (_dest)
50+
if (_renderTexture)
4551
{
46-
var renderText = _dest as RenderTexture;
52+
var renderText = _renderTexture as RenderTexture;
4753
renderText.Release(); // can only be done on main thread
4854
}
4955
}
@@ -57,18 +63,34 @@ protected override bool ReadBuffer()
5763
var textureChanged = false;
5864
try
5965
{
60-
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
66+
if (_renderTexture == null || _renderTexture.width != GetWidth() || _renderTexture.height != GetHeight())
6167
{
6268
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
6369
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
6470
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
6571
_bufferType = GetVideoBufferType(_textureFormat);
66-
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
72+
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
6773
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
74+
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
6875
textureChanged = true;
6976
}
70-
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
71-
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
77+
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);
78+
79+
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
80+
// Flip the texture for OSX
81+
Graphics.CopyTexture(_renderTexture, _dest);
82+
var pixels = _dest.GetPixels();
83+
var flippedPixels = new Color[pixels.Length];
84+
for (int i = 0; i < _dest.height; i++)
85+
{
86+
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
87+
}
88+
_dest.SetPixels(flippedPixels);
89+
#else
90+
Graphics.CopyTexture(_renderTexture, _dest);
91+
#endif
92+
93+
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
7294
}
7395
catch (Exception e)
7496
{

‎Runtime/Scripts/TextureVideoSource.cs

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using UnityEngine.Rendering;
44
using Unity.Collections;
55
using UnityEngine.Experimental.Rendering;
6+
using System;
67

78
namespace LiveKit
89
{
@@ -22,6 +23,11 @@ public override int GetHeight()
2223
return Texture.height;
2324
}
2425

26+
protected override VideoRotation GetVideoRotation()
27+
{
28+
return VideoRotation._0;
29+
}
30+
2531
public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
2632
{
2733
Texture = texture;
@@ -40,31 +46,16 @@ protected override bool ReadBuffer()
4046
return false;
4147
_reading = true;
4248
var textureChanged = false;
43-
if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))
44-
{
45-
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
46-
{
47-
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
48-
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
49-
_bufferType = GetVideoBufferType(_textureFormat);
50-
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
51-
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
52-
textureChanged = true;
53-
}
54-
Graphics.CopyTexture(Texture, _dest);
55-
}
56-
else
57-
{
58-
if(_dest == null || _dest != Texture)
59-
{
60-
textureChanged = true;
61-
}
6249

63-
_dest = Texture;
64-
_textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);
50+
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) {
51+
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
52+
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
6553
_bufferType = GetVideoBufferType(_textureFormat);
54+
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
55+
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
56+
textureChanged = true;
6657
}
67-
58+
Graphics.CopyTexture(Texture, _dest);
6859
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
6960
return textureChanged;
7061
}

‎Runtime/Scripts/WebCameraSource.cs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using UnityEngine;
2+
using LiveKit.Proto;
3+
using UnityEngine.Rendering;
4+
using Unity.Collections;
5+
using UnityEngine.Experimental.Rendering;
6+
using System;
7+
using System.Runtime.InteropServices;
8+
9+
namespace LiveKit
10+
{
11+
// VideoSource for Unity WebCamTexture
12+
public class WebCameraSource : RtcVideoSource
13+
{
14+
TextureFormat _textureFormat;
15+
16+
public WebCamTexture Texture { get; }
17+
18+
public override int GetWidth()
19+
{
20+
return Texture.width;
21+
}
22+
23+
public override int GetHeight()
24+
{
25+
return Texture.height;
26+
}
27+
28+
protected override VideoRotation GetVideoRotation()
29+
{
30+
return VideoRotation._180;
31+
}
32+
33+
public WebCameraSource(WebCamTexture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
34+
{
35+
Texture = texture;
36+
base.Init();
37+
}
38+
39+
~WebCameraSource()
40+
{
41+
Dispose();
42+
}
43+
44+
// Read the texture data into a native array asynchronously
45+
protected override bool ReadBuffer()
46+
{
47+
if (_reading && !Texture.isPlaying)
48+
return false;
49+
_reading = true;
50+
var textureChanged = false;
51+
52+
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
53+
{
54+
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
55+
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
56+
_bufferType = GetVideoBufferType(_textureFormat);
57+
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
58+
_dest = new Texture2D(GetWidth(), GetHeight(), TextureFormat.BGRA32, false);
59+
textureChanged = true;
60+
}
61+
62+
Color32[] pixels = new Color32[GetWidth() * GetHeight()];
63+
Texture.GetPixels32(pixels);
64+
var bytes = MemoryMarshal.Cast<Color32, byte>(pixels);
65+
_data.CopyFrom(bytes.ToArray());
66+
_requestPending = true;
67+
Graphics.CopyTexture(Texture, _dest);
68+
69+
return textureChanged;
70+
}
71+
}
72+
}
73+

‎Runtime/Scripts/WebCameraSource.cs.meta

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)