Skip to content

Commit 420ff74

Browse files
ipavlidakisipavlidakis
authored
[Enhancement]Include the capturer preferredMediaSubtype in the requirements for the capturing format (#881)
1 parent df4c01e commit 420ff74

File tree

7 files changed

+70
-26
lines changed

7 files changed

+70
-26
lines changed

‎Sources/StreamVideo/Utils/AudioSession/Policies/OwnCapabilitiesAudioSessionPolicy.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ public struct OwnCapabilitiesAudioSessionPolicy: AudioSessionPolicy {
2121

2222
@Injected(\.applicationStateAdapter) private var applicationStateAdapter
2323

24-
private let currentDevice = CurrentDevice.currentValue
25-
2624
/// Initializes a new `OwnCapabilitiesAudioSessionPolicy` instance.
2725
public init() {}
2826

@@ -46,7 +44,9 @@ public struct OwnCapabilitiesAudioSessionPolicy: AudioSessionPolicy {
4644
)
4745
}
4846

49-
let currentDeviceHasEarpiece = currentDevice.deviceType == .phone
47+
let currentDeviceHasEarpiece = CurrentDevice
48+
.currentValue
49+
.deviceType == .phone
5050

5151
let category: AVAudioSession.Category = callSettings.audioOn
5252
|| (callSettings.speakerOn && currentDeviceHasEarpiece)

‎Sources/StreamVideo/WebRTC/v2/Extensions/AVFoundation/AVCaptureDevice+OutputFormat.swift

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,27 @@ import Foundation
77
import StreamWebRTC
88

99
extension AVCaptureDevice {
10-
/// Selects an optimal output format for the capture device based on preferred dimensions and frame rate.
10+
/// Selects an optimal output format for the capture device based on preferred dimensions, frame rate, and media subtype.
1111
///
1212
/// - Parameters:
1313
/// - preferredDimensions: The desired video dimensions (width and height in pixels).
1414
/// - preferredFrameRate: The desired frame rate in frames per second.
15-
/// - Returns: An `AVCaptureDevice.Format` instance that closely matches the preferred dimensions
16-
/// and frame rate, or `nil` if no suitable format is found.
15+
/// - preferredMediaSubType: The desired pixel format or media encoding (e.g., '420v').
16+
/// - Returns: An `AVCaptureDevice.Format` instance that closely matches the preferred dimensions,
17+
/// frame rate, and media subtype, or `nil` if no suitable format is found.
1718
///
1819
/// This method evaluates the supported formats for the capture device, using the following criteria:
19-
/// 1. A format with the exact area (dimensions) and frame rate is prioritized.
20-
/// 2. If an exact match is unavailable, a format with a matching area (dimensions) is selected.
21-
/// 3. If no formats match the preferred area, the format with the smallest area difference is chosen.
20+
/// 1. A format with the exact area, frame rate, and media subtype is prioritized.
21+
/// 2. A format with the exact area (dimensions) and frame rate.
22+
/// 3. If an exact match is unavailable, a format with a matching area (dimensions) is selected.
23+
/// 4. If no formats match the preferred area, the format with the smallest area difference is chosen.
2224
///
2325
/// - Note: The formats are sorted by their area difference relative to the preferred dimensions
2426
/// before applying the selection criteria.
2527
func outputFormat(
2628
preferredDimensions: CMVideoDimensions,
27-
preferredFrameRate: Int
29+
preferredFrameRate: Int,
30+
preferredMediaSubType: FourCharCode
2831
) -> AVCaptureDevice.Format? {
2932
// Fetch and sort the supported formats by area difference relative to preferred dimensions.
3033
let formats = RTCCameraVideoCapturer
@@ -33,6 +36,14 @@ extension AVCaptureDevice {
3336

3437
// Try to find the best format based on the criteria, in descending priority order.
3538
if let result = formats.first(
39+
with: [
40+
.mediaSubType(preferredMediaSubType),
41+
.area(preferredDimensions: preferredDimensions),
42+
.frameRate(preferredFrameRate: preferredFrameRate)
43+
]
44+
) {
45+
return result
46+
} else if let result = formats.first(
3647
with: [
3748
.area(preferredDimensions: preferredDimensions),
3849
.frameRate(preferredFrameRate: preferredFrameRate)
@@ -69,12 +80,16 @@ extension Array where Element == AVCaptureDevice.Format {
6980
/// Requires the format to have the smallest area difference relative
7081
/// to the specified `preferredDimensions`.
7182
case minimumAreaDifference(preferredDimensions: CMVideoDimensions)
83+
84+
/// Requires the format to have the specified media subtype.
85+
case mediaSubType(FourCharCode)
7286
}
7387

7488
/// Selects the first format that meets the specified requirements.
7589
///
7690
/// - Parameter requirements: An array of requirements that the format must
77-
/// meet. Requirements are applied in the order they appear.
91+
/// meet. Requirements are applied in the order they appear. This can
92+
/// include resolution, frame rate, media subtype, or area difference.
7893
/// - Returns: The first `AVCaptureDevice.Format` that satisfies all the
7994
/// provided requirements, or `nil` if none match.
8095
///
@@ -107,6 +122,9 @@ extension Array where Element == AVCaptureDevice.Format {
107122
} else {
108123
possibleResults = []
109124
}
125+
126+
case let .mediaSubType(value):
127+
possibleResults = possibleResults.filter { $0.mediaSubType == value }
110128
}
111129
}
112130

‎Sources/StreamVideo/WebRTC/v2/Extensions/AVFoundation/AVCaptureDevice.Format+MediaSubType.swift

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
//
2+
// Copyright © 2025 Stream.io Inc. All rights reserved.
3+
//
4+
5+
import AVFoundation
6+
7+
extension AVCaptureDevice.Format {
8+
/// The media subtype associated with the format description.
9+
///
10+
/// This represents the FourCharCode describing the pixel format or media
11+
/// encoding (e.g., '420v' for NV12). Useful for identifying the format
12+
/// used by a capture device.
13+
var mediaSubType: FourCharCode { CMFormatDescriptionGetMediaSubType(formatDescription) }
14+
}

‎Sources/StreamVideo/WebRTC/v2/PeerConnection/Protocols/CaptureDeviceProtocol.swift

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,23 @@ import AVFoundation
66

77
/// A protocol that defines the properties and methods for a capture device.
88
protocol CaptureDeviceProtocol: Sendable {
9-
/// The position of the capture device.
9+
/// The physical position of the capture device (e.g., front or back).
1010
var position: AVCaptureDevice.Position { get }
1111

12-
/// Returns the output format for the capture device based on the preferred
13-
/// dimensions and frame rate.
12+
/// Returns a suitable output format based on the desired video settings.
13+
///
14+
/// This method searches the available formats on the capture device to find
15+
/// one that matches the given dimensions, frame rate, and media subtype.
1416
///
1517
/// - Parameters:
16-
/// - preferredDimensions: The preferred dimensions for the video output.
17-
/// - preferredFrameRate: The preferred frame rate for the video output.
18-
/// - Returns: An optional `AVCaptureDevice.Format` that matches the
19-
/// preferred dimensions and frame rate.
18+
/// - preferredDimensions: Desired width and height of the video.
19+
/// - preferredFrameRate: Desired number of frames per second.
20+
/// - preferredMediaSubType: Desired pixel format or encoding (e.g., '420v').
21+
/// - Returns: A format that best matches the criteria, or `nil` if none found.
2022
func outputFormat(
2123
preferredDimensions: CMVideoDimensions,
22-
preferredFrameRate: Int
24+
preferredFrameRate: Int,
25+
preferredMediaSubType: FourCharCode
2326
) -> AVCaptureDevice.Format?
2427
}
2528

‎Sources/StreamVideo/WebRTC/v2/VideoCapturing/ActionHandlers/Camera/CameraCaptureHandler.swift

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
//
44

55
import AVFoundation
6+
import Combine
67
import CoreMedia
78
import Foundation
89
import StreamWebRTC
@@ -115,7 +116,8 @@ final class CameraCaptureHandler: StreamVideoCapturerActionHandler, @unchecked S
115116

116117
guard let outputFormat = captureDevice.outputFormat(
117118
preferredDimensions: .init(configuration.dimensions),
118-
preferredFrameRate: configuration.frameRate
119+
preferredFrameRate: configuration.frameRate,
120+
preferredMediaSubType: videoCapturer.preferredOutputPixelFormat()
119121
) else {
120122
throw ClientError(
121123
"\(type(of: self)) was unable to perform action because no output format found for dimensions:\(configuration.dimensions) frameRate:\(configuration.frameRate)."
@@ -215,7 +217,8 @@ final class CameraCaptureHandler: StreamVideoCapturerActionHandler, @unchecked S
215217

216218
guard let outputFormat = captureDevice.outputFormat(
217219
preferredDimensions: .init(configuration.dimensions),
218-
preferredFrameRate: configuration.frameRate
220+
preferredFrameRate: configuration.frameRate,
221+
preferredMediaSubType: videoCapturer.preferredOutputPixelFormat()
219222
) else {
220223
throw ClientError(
221224
"\(type(of: self)) was unable to perform action because no output format found for dimensions:\(configuration.dimensions) frameRate:\(configuration.frameRate)."

‎StreamVideo.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,7 @@
363363
408721F12E12741F006A68CB /* DispatchWorkItem+TimerControl.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408721F02E12741F006A68CB /* DispatchWorkItem+TimerControl.swift */; };
364364
408721F72E127551006A68CB /* TimerPublisher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408721F62E127551006A68CB /* TimerPublisher.swift */; };
365365
408721FA2E127A18006A68CB /* TimerPublisher_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408721F92E127A18006A68CB /* TimerPublisher_Tests.swift */; };
366+
408722372E13C91F006A68CB /* AVCaptureDevice.Format+MediaSubType.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408722362E13C91F006A68CB /* AVCaptureDevice.Format+MediaSubType.swift */; };
366367
4089378B2C062B17000EEB69 /* StreamUUIDFactory.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4089378A2C062B17000EEB69 /* StreamUUIDFactory.swift */; };
367368
408937912C134305000EEB69 /* View+AlertWithTextField.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408937902C134305000EEB69 /* View+AlertWithTextField.swift */; };
368369
408CE0F72BD95EB60052EC3A /* VideoConfig+Dummy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */; };
@@ -1937,6 +1938,7 @@
19371938
408721F02E12741F006A68CB /* DispatchWorkItem+TimerControl.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DispatchWorkItem+TimerControl.swift"; sourceTree = "<group>"; };
19381939
408721F62E127551006A68CB /* TimerPublisher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimerPublisher.swift; sourceTree = "<group>"; };
19391940
408721F92E127A18006A68CB /* TimerPublisher_Tests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimerPublisher_Tests.swift; sourceTree = "<group>"; };
1941+
408722362E13C91F006A68CB /* AVCaptureDevice.Format+MediaSubType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Format+MediaSubType.swift"; sourceTree = "<group>"; };
19401942
4089378A2C062B17000EEB69 /* StreamUUIDFactory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamUUIDFactory.swift; sourceTree = "<group>"; };
19411943
408937902C134305000EEB69 /* View+AlertWithTextField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "View+AlertWithTextField.swift"; sourceTree = "<group>"; };
19421944
408CE0F62BD95EB60052EC3A /* VideoConfig+Dummy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VideoConfig+Dummy.swift"; sourceTree = "<group>"; };
@@ -5050,6 +5052,7 @@
50505052
40D36ACF2DDDF97500972D75 /* AVAudioRecorder+Sendable.swift */,
50515053
40AAD17D2D27FBD800D10330 /* AVCaptureDevice.Position+Convenience.swift */,
50525054
40E363512D0A11620028C52A /* AVCaptureDevice+OutputFormat.swift */,
5055+
408722362E13C91F006A68CB /* AVCaptureDevice.Format+MediaSubType.swift */,
50535056
40E363442D09F2BD0028C52A /* AVCaptureDevice.Format+Convenience.swift */,
50545057
);
50555058
path = AVFoundation;
@@ -8126,6 +8129,7 @@
81268129
848CCCE82AB8ED8F002E83A2 /* StartHLSBroadcastingResponse.swift in Sources */,
81278130
84D2E37629DC856D001D2118 /* CallMemberRemovedEvent.swift in Sources */,
81288131
40E3636C2D0A24390028C52A /* ScreenShareCaptureHandler.swift in Sources */,
8132+
408722372E13C91F006A68CB /* AVCaptureDevice.Format+MediaSubType.swift in Sources */,
81298133
84F73859287C1A3400A363F4 /* StreamVideo.swift in Sources */,
81308134
4012B1892BFC9E6F006B0031 /* UnfairQueue.swift in Sources */,
81318135
840F59922A77FDCB00EF3EB2 /* UnpinResponse.swift in Sources */,

‎StreamVideoTests/Mock/MockCaptureDevice.swift

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ final class MockCaptureDevice: CaptureDeviceProtocol, Mockable, @unchecked Senda
2828
}
2929

3030
enum MockFunctionInputKey: Payloadable {
31-
case outputFormat(preferredDimensions: CMVideoDimensions, preferredFrameRate: Int)
31+
case outputFormat(preferredDimensions: CMVideoDimensions, preferredFrameRate: Int, preferredMediaSubType: FourCharCode)
3232

3333
var payload: Any {
3434
switch self {
35-
case let .outputFormat(preferredDimensions, preferredFrameRate):
36-
return (preferredDimensions, preferredFrameRate)
35+
case let .outputFormat(preferredDimensions, preferredFrameRate, preferredMediaSubType):
36+
return (preferredDimensions, preferredFrameRate, preferredMediaSubType)
3737
}
3838
}
3939
}
@@ -45,12 +45,14 @@ final class MockCaptureDevice: CaptureDeviceProtocol, Mockable, @unchecked Senda
4545

4646
func outputFormat(
4747
preferredDimensions: CMVideoDimensions,
48-
preferredFrameRate: Int
48+
preferredFrameRate: Int,
49+
preferredMediaSubType: FourCharCode
4950
) -> AVCaptureDevice.Format? {
5051
stubbedFunctionInput[.outputFormat]?.append(
5152
.outputFormat(
5253
preferredDimensions: preferredDimensions,
53-
preferredFrameRate: preferredFrameRate
54+
preferredFrameRate: preferredFrameRate,
55+
preferredMediaSubType: preferredMediaSubType
5456
)
5557
)
5658

0 commit comments

Comments
 (0)