diff --git a/Sources/LiveKit/Protocols/VideoRenderer.swift b/Sources/LiveKit/Protocols/VideoRenderer.swift index bc65c078f..42193d33f 100644 --- a/Sources/LiveKit/Protocols/VideoRenderer.swift +++ b/Sources/LiveKit/Protocols/VideoRenderer.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import AVFoundation import Foundation @_implementationOnly import LiveKitWebRTC @@ -30,25 +31,36 @@ public protocol VideoRenderer { var adaptiveStreamSize: CGSize { get } /// Size of the frame. + @objc optional func set(size: CGSize) + @objc optional func render(frame: VideoFrame) + + // Only invoked for local tracks, provides additional capture time options + @objc optional + func render(frame: VideoFrame, videoCaptureOptions: VideoCaptureOptions?) } class VideoRendererAdapter: NSObject, LKRTCVideoRenderer { private weak var target: VideoRenderer? + private weak var localVideoTrack: LocalVideoTrack? - init(target: VideoRenderer) { + init(target: VideoRenderer, localVideoTrack: LocalVideoTrack?) { self.target = target + self.localVideoTrack = localVideoTrack } func setSize(_ size: CGSize) { - target?.set(size: size) + target?.set?(size: size) } func renderFrame(_ frame: LKRTCVideoFrame?) { guard let frame = frame?.toLKType() else { return } - target?.render(frame: frame) + target?.render?(frame: frame) + + let cameraCapturer = localVideoTrack?.capturer as? CameraCapturer + target?.render?(frame: frame, videoCaptureOptions: cameraCapturer?.options) } // Proxy the equality operators diff --git a/Sources/LiveKit/Track/Track.swift b/Sources/LiveKit/Track/Track.swift index dcda5c166..d67e02119 100644 --- a/Sources/LiveKit/Track/Track.swift +++ b/Sources/LiveKit/Track/Track.swift @@ -370,7 +370,7 @@ extension Track { } videoRenderers.add(videoRenderer) - rtcVideoTrack.add(VideoRendererAdapter(target: videoRenderer)) + rtcVideoTrack.add(VideoRendererAdapter(target: videoRenderer, localVideoTrack: self as? LocalVideoTrack)) } func _remove(videoRenderer: VideoRenderer) { @@ -384,7 +384,7 @@ extension Track { } videoRenderers.remove(videoRenderer) - rtcVideoTrack.remove(VideoRendererAdapter(target: videoRenderer)) + rtcVideoTrack.remove(VideoRendererAdapter(target: videoRenderer, localVideoTrack: self as? LocalVideoTrack)) } } diff --git a/Sources/LiveKit/Types/Options/CameraCaptureOptions.swift b/Sources/LiveKit/Types/Options/CameraCaptureOptions.swift index 414335925..af01295d8 100644 --- a/Sources/LiveKit/Types/Options/CameraCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/CameraCaptureOptions.swift @@ -82,4 +82,10 @@ public class CameraCaptureOptions: NSObject, VideoCaptureOptions { hasher.combine(fps) return hasher.finalize() } + + // MARK: - CustomStringConvertible + + override public var description: String { + "CameraCaptureOptions(position: \(String(describing: position))" + } } diff --git a/Sources/LiveKit/Views/VideoView.swift b/Sources/LiveKit/Views/VideoView.swift index cc0ba6580..1d848dca2 100644 --- a/Sources/LiveKit/Views/VideoView.swift +++ b/Sources/LiveKit/Views/VideoView.swift @@ -153,7 +153,7 @@ public class VideoView: NativeView, Loggable { // MARK: - Internal - struct State: Equatable { + struct State { weak var track: Track? var isEnabled: Bool = true var isHidden: Bool = false @@ -174,6 +174,9 @@ public class VideoView: NativeView, Loggable { var didRenderFirstFrame: Bool = false var isRendering: Bool = false + // Only used for rendering local tracks + var videoCaptureOptions: VideoCaptureOptions? = nil + // whether if current state should be rendering var shouldRender: Bool { track != nil && isEnabled && !isHidden @@ -236,11 +239,6 @@ public class VideoView: NativeView, Loggable { nr.removeFromSuperview() self._nativeRenderer = nil } - - // CapturerDelegate - if let localTrack = track as? LocalVideoTrack { - localTrack.capturer.remove(delegate: self) - } } // set new track @@ -256,11 +254,6 @@ public class VideoView: NativeView, Loggable { nr.renderFrame(frame.toRTCType()) self.setNeedsLayout() } - - // CapturerDelegate - if let localTrack = track as? LocalVideoTrack { - localTrack.capturer.add(delegate: self) - } } } @@ -290,6 +283,13 @@ public class VideoView: NativeView, Loggable { // https://developer.apple.com/forums/thread/105252 // nativeRenderer.asMetalView?.isPaused = !shouldAttach + var capturePositionDidUpdate = false + if let oldCameraCaptureOptions = oldState.videoCaptureOptions as? CameraCaptureOptions, + let newCameraCaptureOptions = newState.videoCaptureOptions as? CameraCaptureOptions + { + capturePositionDidUpdate = oldCameraCaptureOptions.position != newCameraCaptureOptions.position + } + // layout is required if any of the following vars mutate if newState.isDebugMode != oldState.isDebugMode || newState.layoutMode != oldState.layoutMode || @@ -297,7 +297,7 @@ public class VideoView: NativeView, Loggable { newState.renderMode != oldState.renderMode || newState.rotationOverride != oldState.rotationOverride || newState.didRenderFirstFrame != oldState.didRenderFirstFrame || - shouldRenderDidUpdate || trackDidUpdate + shouldRenderDidUpdate || trackDidUpdate || capturePositionDidUpdate { // must be on main Task.detached { @MainActor in @@ -444,7 +444,7 @@ public class VideoView: NativeView, Loggable { } } - _nativeRenderer.set(mirrored: shouldMirror()) + _nativeRenderer.set(mirrored: _shouldMirror()) } } @@ -488,12 +488,11 @@ private extension VideoView { return newView } - func shouldMirror() -> Bool { + func _shouldMirror() -> Bool { switch _state.mirrorMode { case .auto: - guard let localVideoTrack = _state.track as? LocalVideoTrack, - let cameraCapturer = localVideoTrack.capturer as? CameraCapturer, - case .front = cameraCapturer.options.position else { return false } + guard let cameraCaptureOptions = _state.videoCaptureOptions as? CameraCaptureOptions, + case .front = cameraCaptureOptions.position else { return false } return true case .off: return false case .mirror: return true @@ -519,7 +518,7 @@ extension VideoView: VideoRenderer { } } - public func render(frame: VideoFrame) { + public func render(frame: VideoFrame, videoCaptureOptions: VideoCaptureOptions?) { let state = _state.copy() // prevent any extra rendering if already !isEnabled etc. @@ -556,6 +555,7 @@ extension VideoView: VideoRenderer { track?.set(videoFrame: frame) _state.mutate { + $0.videoCaptureOptions = videoCaptureOptions $0.didRenderFirstFrame = true $0.isRendering = true $0.renderDate = Date() @@ -569,18 +569,6 @@ extension VideoView: VideoRenderer { } } -// MARK: - VideoCapturerDelegate - -extension VideoView: VideoCapturerDelegate { - public func capturer(_: VideoCapturer, didUpdate state: VideoCapturer.CapturerState) { - if case .started = state { - Task.detached { @MainActor in - self.setNeedsLayout() - } - } - } -} - // MARK: - Internal extension VideoView {