From ba75e79c2d36f947f1b2d7822b87a16fe707aa7e Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Wed, 26 Feb 2020 15:15:00 +0000 Subject: [PATCH] Add transceiver API to C++ This is part 1/N of the change, see global description below ---- Add standalone media track objects in both the C++ and C# library. This enables supporting multiple audio and video tracks per peer connection. For local tracks, the `PeerConnection` interface is updated such that the functionality of `AddLocalAudioTrackAsync()` and `AddLocalVideoTrackAsync()` is split in 2 steps: - the creation step, whose interface is on the local track object itself, creates a standalone track object not associated with any peer connection and not currently used by anything. - the association step, which takes the track and attach it to a peer connection via a transceiver. For remote tracks, the `TrackAdded` and `TrackRemoved` events now generate a new remote track object, which is passed to the event handler as argument. The C# `PeerConnection` object also now keeps a list of all the tracks attached to it, which can therefore be enumerated. Because tracks are standalone objects, methods like `PeerConnection.SetLocalAudioTrackEnabled()` are replaced with a property `LocalAudioTrack.Enabled`, providing a more familiar pattern to C# developers. One important change required for remote tracks to be paired by name is removing the legacy options `RTCOfferAnswerOptions::offer_to_receive_video = true` and `RTCOfferAnswerOptions::offer_to_receive_audio = true` from `CreateOffer()` and `CreateAnswer()`. This change (see #75) has larger implications that it looks when using Unified Plan (default), as previously those options were implicitly making an offer to receive some media, whereas this step is now explicit and the user needs to add some transceiver in receive-only or send-receive mode before making an offer/answer to allow the remote peer to send. A new Unity demo `StandaloneDemo` shows how to use multiple tracks between two peers. The demo is standalone in the sense signaling is hard-coded and does not require an external node-dss server or any other signaling solution. Bug: #152 --- .../include/interop_api.h | 469 ++++-- .../include/local_video_track_interop.h | 15 + .../include/peer_connection_interop.h | 14 + .../src/interop/audio_transceiver_interop.cpp | 87 + .../src/interop/audio_transceiver_interop.h | 56 + .../src/interop/global_factory.cpp | 16 +- .../src/interop/interop_api.cpp | 276 ++-- .../src/interop/local_audio_track_interop.cpp | 61 + .../src/interop/local_audio_track_interop.h | 50 + .../src/interop/local_video_track_interop.cpp | 51 + .../src/interop/peer_connection_interop.cpp | 40 + .../interop/remote_audio_track_interop.cpp | 60 + .../src/interop/remote_audio_track_interop.h | 43 + .../interop/remote_video_track_interop.cpp | 69 + .../src/interop/remote_video_track_interop.h | 51 + .../src/interop/video_transceiver_interop.cpp | 87 + .../src/interop/video_transceiver_interop.h | 56 + .../src/media/audio_transceiver.cpp | 144 ++ .../src/media/audio_transceiver.h | 73 + .../src/media/local_audio_track.cpp | 117 ++ .../src/media/local_audio_track.h | 116 ++ .../src/media/local_video_track.cpp | 69 +- .../src/media/local_video_track.h | 40 +- .../src/media/media_track.cpp | 27 + .../src/media/media_track.h | 47 + .../src/media/remote_audio_track.cpp | 63 + .../src/media/remote_audio_track.h | 89 ++ .../src/media/remote_video_track.cpp | 73 + .../src/media/remote_video_track.h | 89 ++ .../src/media/transceiver.cpp | 142 ++ .../src/media/transceiver.h | 132 ++ .../src/media/video_transceiver.cpp | 143 ++ .../src/media/video_transceiver.h | 73 + .../src/peer_connection.cpp | 1410 ++++++++++++++--- .../src/peer_connection.h | 206 ++- .../src/tracked_object.h | 6 + .../src/utils.h | 60 +- ...oft.MixedReality.WebRTC.Native.UWP.vcxproj | 24 + ...dReality.WebRTC.Native.UWP.vcxproj.filters | 72 + ...t.MixedReality.WebRTC.Native.Win32.vcxproj | 24 + ...eality.WebRTC.Native.Win32.vcxproj.filters | 72 + ...t.MixedReality.WebRTC.Native.Tests.vcxproj | 4 + .../test/audio_track_tests.cpp | 261 ++- .../test/audio_transceiver_tests.cpp | 217 +++ .../external_video_track_source_tests.cpp | 151 +- .../test/pch.h | 2 + .../test/peer_connection_test_helpers.h | 5 + .../test/peer_connection_tests.cpp | 3 +- .../test/simple_interop.cpp | 114 ++ .../test/simple_interop.h | 56 + .../test/video_test_utils.h | 4 +- .../test/video_track_tests.cpp | 524 +++++- .../test/video_transceiver_tests.cpp | 612 +++++++ 53 files changed, 5971 insertions(+), 794 deletions(-) create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h create mode 100644 libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h index 31df2c7ff..444e2a8dc 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h @@ -70,23 +70,112 @@ MRS_API void MRS_CALL mrsCloseEnum(mrsEnumHandle* handleRef) noexcept; // Interop // +struct mrsAudioTransceiverConfig; +struct mrsVideoTransceiverConfig; +struct mrsRemoteAudioTrackConfig; +struct mrsRemoteVideoTrackConfig; struct mrsDataChannelConfig; struct mrsDataChannelCallbacks; +/// Opaque handle to a native PeerConnection C++ object. +using PeerConnectionHandle = void*; + +/// Opaque handle to a native MediaTrack C++ object. +using MediaTrackHandle = void*; + +/// Opaque handle to a native AudioTransceiver C++ object. +using AudioTransceiverHandle = void*; + +/// Opaque handle to a native VideoTransceiver C++ object. +using VideoTransceiverHandle = void*; + +/// Opaque handle to a native LocalAudioTrack C++ object. +using LocalAudioTrackHandle = void*; + +/// Opaque handle to a native LocalVideoTrack C++ object. +using LocalVideoTrackHandle = void*; + +/// Opaque handle to a native RemoteAudioTrack C++ object. +using RemoteAudioTrackHandle = void*; + +/// Opaque handle to a native RemoteVideoTrack C++ object. +using RemoteVideoTrackHandle = void*; + +/// Opaque handle to a native DataChannel C++ object. +using DataChannelHandle = void*; + +/// Opaque handle to a native ExternalVideoTrackSource C++ object. +using ExternalVideoTrackSourceHandle = void*; + /// Opaque handle to the interop wrapper of a peer connection. using mrsPeerConnectionInteropHandle = void*; +/// Opaque handle to the interop wrapper of an audio transceiver. +using mrsAudioTransceiverInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a video transceiver. +using mrsVideoTransceiverInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a local audio track. +using mrsLocalAudioTrackInteropHandle = void*; + /// Opaque handle to the interop wrapper of a local video track. using mrsLocalVideoTrackInteropHandle = void*; +/// Opaque handle to the interop wrapper of a remote audio track. +using mrsRemoteAudioTrackInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a remote video track. +using mrsRemoteVideoTrackInteropHandle = void*; + /// Opaque handle to the interop wrapper of a data channel. using mrsDataChannelInteropHandle = void*; +/// Callback to create an interop wrapper for an audio transceiver. +/// The callback must return the handle of the created interop wrapper. +using mrsAudioTransceiverCreateObjectCallback = + mrsAudioTransceiverInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsAudioTransceiverConfig& config) noexcept; + +/// Callback to finish the creation of the interop wrapper by assigning to it +/// the handle of the AudioTransceiver native object it wraps. +/// This is called shortly after |mrsAudioTransceiverCreateObjectCallback|, with +/// the same |mrsAudioTransceiverInteropHandle| returned by that callback. +using mrsAudioTransceiverFinishCreateCallback = + void(MRS_CALL*)(mrsAudioTransceiverInteropHandle, AudioTransceiverHandle); + +/// Callback to create an interop wrapper for a video transceiver. +/// The callback must return the handle of the created interop wrapper. +using mrsVideoTransceiverCreateObjectCallback = + mrsVideoTransceiverInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsVideoTransceiverConfig& config) noexcept; + +/// Callback to finish the creation of the interop wrapper by assigning to it +/// the handle of the VideoTransceiver native object it wraps. +/// This is called shortly after |mrsVideoTransceiverCreateObjectCallback|, with +/// the same |mrsVideoTransceiverInteropHandle| returned by that callback. +using mrsVideoTransceiverFinishCreateCallback = + void(MRS_CALL*)(mrsVideoTransceiverInteropHandle, VideoTransceiverHandle); + +/// Callback to create an interop wrapper for a remote audio track. +using mrsRemoteAudioTrackCreateObjectCallback = + mrsRemoteAudioTrackInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + +/// Callback to create an interop wrapper for a remote video track. +using mrsRemoteVideoTrackCreateObjectCallback = + mrsRemoteVideoTrackInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + /// Callback to create an interop wrapper for a data channel. using mrsDataChannelCreateObjectCallback = mrsDataChannelInteropHandle( MRS_CALL*)(mrsPeerConnectionInteropHandle parent, const mrsDataChannelConfig& config, - mrsDataChannelCallbacks* callbacks); + mrsDataChannelCallbacks* callbacks) noexcept; // // Video capture enumeration @@ -141,18 +230,6 @@ MRS_API mrsResult MRS_CALL mrsEnumVideoCaptureFormatsAsync( // Peer connection // -/// Opaque handle to a native PeerConnection C++ object. -using PeerConnectionHandle = void*; - -/// Opaque handle to a native LocalVideoTrack C++ object. -using LocalVideoTrackHandle = void*; - -/// Opaque handle to a native DataChannel C++ object. -using DataChannelHandle = void*; - -/// Opaque handle to a native ExternalVideoTrackSource C++ object. -using ExternalVideoTrackSourceHandle = void*; - /// Callback fired when the peer connection is connected, that is it finished /// the JSEP offer/answer exchange successfully. using PeerConnectionConnectedCallback = void(MRS_CALL*)(void* user_data); @@ -209,24 +286,64 @@ enum class TrackKind : uint32_t { kDataTrack = 3, }; -/// Callback fired when a remote track is added to a connection. -using PeerConnectionTrackAddedCallback = void(MRS_CALL*)(void* user_data, - TrackKind track_kind); - -/// Callback fired when a remote track is removed from a connection. -using PeerConnectionTrackRemovedCallback = - void(MRS_CALL*)(void* user_data, TrackKind track_kind); +/// Callback fired when a remote audio track is added to a connection. +/// The |audio_track| and |audio_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalAudioTrackRemoveRef()| and +/// |mrsAudioTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionAudioTrackAddedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteAudioTrackInteropHandle audio_track_wrapper, + RemoteAudioTrackHandle audio_track, + mrsAudioTransceiverInteropHandle audio_transceiver_wrapper, + AudioTransceiverHandle audio_transceiver); + +/// Callback fired when a remote audio track is removed from a connection. +/// The |audio_track| and |audio_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalAudioTrackRemoveRef()| and +/// |mrsAudioTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionAudioTrackRemovedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteAudioTrackInteropHandle audio_track_wrapper, + RemoteAudioTrackHandle audio_track, + mrsAudioTransceiverInteropHandle audio_transceiver_wrapper, + AudioTransceiverHandle audio_transceiver); + +/// Callback fired when a remote video track is added to a connection. +/// The |video_track| and |video_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalVideoTrackRemoveRef()| and +/// |mrsVideoTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionVideoTrackAddedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteVideoTrackInteropHandle video_track_wrapper, + RemoteVideoTrackHandle video_track, + mrsVideoTransceiverInteropHandle video_transceiver_wrapper, + VideoTransceiverHandle video_transceiver); + +/// Callback fired when a remote video track is removed from a connection. +/// The |video_track| and |video_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalVideoTrackRemoveRef()| and +/// |mrsVideoTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionVideoTrackRemovedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteVideoTrackInteropHandle video_track_wrapper, + RemoteVideoTrackHandle video_track, + mrsVideoTransceiverInteropHandle video_transceiver_wrapper, + VideoTransceiverHandle video_transceiver); /// Callback fired when a data channel is added to the peer connection after /// being negotiated with the remote peer. using PeerConnectionDataChannelAddedCallback = - void(MRS_CALL*)(void* user_data, + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, mrsDataChannelInteropHandle data_channel_wrapper, DataChannelHandle data_channel); /// Callback fired when a data channel is remoted from the peer connection. using PeerConnectionDataChannelRemovedCallback = - void(MRS_CALL*)(void* user_data, + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, mrsDataChannelInteropHandle data_channel_wrapper, DataChannelHandle data_channel); @@ -250,8 +367,8 @@ using mrsAudioFrame = Microsoft::MixedReality::WebRTC::AudioFrame; /// Callback fired when a local or remote (depending on use) audio frame is /// available to be consumed by the caller, usually for local output. -using PeerConnectionAudioFrameCallback = - void(MRS_CALL*)(void* user_data, const mrsAudioFrame& frame); +using mrsAudioFrameCallback = void(MRS_CALL*)(void* user_data, + const mrsAudioFrame& frame); /// Callback fired when a message is received on a data channel. using mrsDataChannelMessageCallback = void(MRS_CALL*)(void* user_data, @@ -331,37 +448,67 @@ mrsPeerConnectionCreate(PeerConnectionConfiguration config, mrsPeerConnectionInteropHandle interop_handle, PeerConnectionHandle* peerHandleOut) noexcept; +/// Callbacks needed to allow the native implementation to interact with the +/// interop layer, and in particular to react to events which necessitate +/// creating a new interop wrapper for a new native instance (whose creation was +/// not initiated by the interop, so for which the native instance is created +/// first). struct mrsPeerConnectionInteropCallbacks { + /// Construct an interop object for an AudioTransceiver instance. + mrsAudioTransceiverCreateObjectCallback audio_transceiver_create_object{}; + + /// Finish the construction of the interop object of an AudioTransceiver. + mrsAudioTransceiverFinishCreateCallback audio_transceiver_finish_create{}; + + /// Construct an interop object for a VideoTransceiver instance. + mrsVideoTransceiverCreateObjectCallback video_transceiver_create_object{}; + + /// Finish the construction of the interop object of a VideoTransceiver. + mrsVideoTransceiverFinishCreateCallback video_transceiver_finish_create{}; + + /// Construct an interop object for a RemoteAudioTrack instance. + mrsRemoteAudioTrackCreateObjectCallback remote_audio_track_create_object{}; + + /// Construct an interop object for a RemoteVideooTrack instance. + mrsRemoteVideoTrackCreateObjectCallback remote_video_track_create_object{}; + /// Construct an interop object for a DataChannel instance. mrsDataChannelCreateObjectCallback data_channel_create_object{}; }; +/// Register the interop callbacks necessary to make interop work. To +/// unregister, simply pass nullptr as the callback pointer. Only one set of +/// callbacks can be registered at a time. MRS_API mrsResult MRS_CALL mrsPeerConnectionRegisterInteropCallbacks( PeerConnectionHandle peerHandle, mrsPeerConnectionInteropCallbacks* callbacks) noexcept; -/// Register a callback fired once connected to a remote peer. -/// To unregister, simply pass nullptr as the callback pointer. +/// Register a callback invoked once connected to a remote peer. To unregister, +/// simply pass nullptr as the callback pointer. Only one callback can be +/// registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterConnectedCallback( PeerConnectionHandle peerHandle, PeerConnectionConnectedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a local message is ready to be sent via the -/// signaling service to a remote peer. +/// Register a callback invoked when a local message is ready to be sent via the +/// signaling service to a remote peer. Only one callback can be registered at a +/// time. MRS_API void MRS_CALL mrsPeerConnectionRegisterLocalSdpReadytoSendCallback( PeerConnectionHandle peerHandle, PeerConnectionLocalSdpReadytoSendCallback callback, void* user_data) noexcept; -/// Register a callback fired when an ICE candidate message is ready to be sent -/// via the signaling service to a remote peer. +/// Register a callback invoked when an ICE candidate message is ready to be +/// sent via the signaling service to a remote peer. Only one callback can be +/// registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterIceCandidateReadytoSendCallback( PeerConnectionHandle peerHandle, PeerConnectionIceCandidateReadytoSendCallback callback, void* user_data) noexcept; -/// Register a callback fired when the ICE connection state changes. +/// Register a callback invoked when the ICE connection state changes. Only one +/// callback can be registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterIceStateChangedCallback( PeerConnectionHandle peerHandle, PeerConnectionIceStateChangedCallback callback, @@ -374,18 +521,44 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterRenegotiationNeededCallback( PeerConnectionRenegotiationNeededCallback callback, void* user_data) noexcept; -/// Register a callback fired when a remote media track is added to the current +/// Register a callback fired when a remote audio track is added to the current /// peer connection. -MRS_API void MRS_CALL mrsPeerConnectionRegisterTrackAddedCallback( +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterAudioTrackAddedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackAddedCallback callback, + PeerConnectionAudioTrackAddedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a remote media track is removed from the +/// Register a callback fired when a remote audio track is removed from the /// current peer connection. -MRS_API void MRS_CALL mrsPeerConnectionRegisterTrackRemovedCallback( +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterAudioTrackRemovedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackRemovedCallback callback, + PeerConnectionAudioTrackRemovedCallback callback, + void* user_data) noexcept; + +/// Register a callback fired when a remote video track is added to the current +/// peer connection. +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterVideoTrackAddedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackAddedCallback callback, + void* user_data) noexcept; + +/// Register a callback fired when a remote video track is removed from the +/// current peer connection. +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterVideoTrackRemovedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackRemovedCallback callback, void* user_data) noexcept; /// Register a callback fired when a remote data channel is removed from the @@ -402,20 +575,6 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterDataChannelRemovedCallback( PeerConnectionDataChannelRemovedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a video frame from a video track was received -/// from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsI420AVideoFrameCallback callback, - void* user_data) noexcept; - -/// Register a callback fired when a video frame from a video track was received -/// from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsArgb32VideoFrameCallback callback, - void* user_data) noexcept; - /// Kind of video profile. Equivalent to org::webRtc::VideoProfileKind. enum class VideoProfileKind : int32_t { kUnspecified, @@ -431,24 +590,77 @@ enum class VideoProfileKind : int32_t { kVideoHdr8, }; -/// Register a callback fired when an audio frame is available from a local -/// audio track, usually from a local audio capture device (local microphone). -/// -/// -- WARNING -- -/// Currently this callback is never fired, because the internal audio capture -/// device implementation ignores any registration and only delivers its audio -/// data to the internal WebRTC engine for sending to the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterLocalAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept; +enum class mrsTransceiverStateUpdatedReason : int32_t { + kLocalDesc, + kRemoteDesc, + kSetDirection +}; -/// Register a callback fired when an audio frame from an audio track was -/// received from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterRemoteAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept; +/// Flow direction of the media inside the transceiver. This maps to whether +/// local and/or remote tracks are attached to the transceiver. The local +/// track corresponds to the send direction, and the remote track to the +/// receive direction. +enum class mrsTransceiverDirection : int32_t { + kSendRecv = 0, + kSendOnly = 1, + kRecvOnly = 2, + kInactive = 3 +}; + +/// Same as |mrsTransceiverDirection|, but including optional unset. +enum class mrsTransceiverOptDirection : int32_t { + kNotSet = -1, + kSendRecv = 0, + kSendOnly = 1, + kRecvOnly = 2, + kInactive = 3 +}; + +/// Configuration for creating a new audio transceiver. +struct AudioTransceiverInitConfig { + /// Mandatory name of the audio transceiver. This must be a valid SDP token; + /// see |mrsSdpIsValidToken()|. + const char* name = nullptr; + + /// Initial desired direction of the transceiver media when created. + mrsTransceiverDirection desired_direction = + mrsTransceiverDirection::kSendRecv; + + /// Semi-colon separated list of stream IDs associated with the transceiver. + /// Use |Transceiver::DecodeStreamIDs()| and |Transceiver::EncodeStreamIDs()|. + const char* stream_ids = nullptr; + + /// Handle of the audio transceiver interop wrapper, if any, which will be + /// associated with the native audio transceiver object. + mrsAudioTransceiverInteropHandle transceiver_interop_handle{}; +}; + +/// Configuration for creating a new video transceiver. +struct VideoTransceiverInitConfig { + /// Mandatory name of the video transceiver. This must be a valid SDP token; + /// see |mrsSdpIsValidToken()|. + const char* name = nullptr; + + /// Initial desired direction of the transceiver media when created. + mrsTransceiverDirection desired_direction = + mrsTransceiverDirection::kSendRecv; + + /// Semi-colon separated list of stream IDs associated with the transceiver. + /// Use |Transceiver::DecodeStreamIDs()| and |Transceiver::EncodeStreamIDs()|. + const char* stream_ids = nullptr; + + /// Handle of the video transceiver interop wrapper, if any, which will be + /// associated with the native video transceiver object. + mrsVideoTransceiverInteropHandle transceiver_interop_handle{}; +}; + +/// Configuration for opening a local audio capture device and creating a local +/// audio track. +struct LocalAudioTrackInitConfig { + /// Handle of the local audio track interop wrapper, if any, which will be + /// associated with the native local audio track object. + mrsLocalAudioTrackInteropHandle track_interop_handle{}; +}; /// Configuration for opening a local video capture device and creating a local /// video track. @@ -502,19 +714,12 @@ struct LocalVideoTrackInitConfig { mrsBool enable_mrc_recording_indicator = mrsBool::kTrue; }; -/// Add a local video track from a local video capture device (webcam) to -/// the collection of tracks to send to the remote peer. -/// |video_device_id| specifies the unique identifier of a video capture -/// device to open, as obtained by enumerating devices with -/// mrsEnumVideoCaptureDevicesAsync(), or null for any device. -/// |enable_mrc| allows enabling Mixed Reality Capture on HoloLens devices, and -/// is otherwise ignored for other video capture devices. On UWP this must be -/// invoked from another thread than the main UI thread. -MRS_API mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( - PeerConnectionHandle peerHandle, - const char* track_name, - const LocalVideoTrackInitConfig* config, - LocalVideoTrackHandle* trackHandle) noexcept; +/// Configuration for creating a local video track from an external source. +struct LocalVideoTrackFromExternalSourceInitConfig { + /// Handle of the local video track interop wrapper, if any, which will be + /// associated with the native local video track object. + mrsLocalVideoTrackInteropHandle track_interop_handle{}; +}; using mrsRequestExternalI420AVideoFrameCallback = mrsResult(MRS_CALL*)(void* user_data, @@ -528,45 +733,57 @@ using mrsRequestExternalArgb32VideoFrameCallback = uint32_t request_id, int64_t timestamp_ms); -/// Configuration for creating a local video track from an external source. -struct LocalVideoTrackFromExternalSourceInitConfig { - /// Handle of the local video track interop wrapper, if any, which will be - /// associated with the native local video track object. - mrsLocalVideoTrackInteropHandle track_interop_handle{}; +/// Configuration for creating a new audio transceiver interop wrapper. +struct mrsAudioTransceiverConfig { + /// Transceiver name, for pairing with remote peer's one. This is normally + /// extracted from the receiver's stream ID #0 if the transceiver was created + /// for a remote track, or is determined by the user-provided value if created + /// with |AddAudioTransceiver()|. + const char* name{}; + + /// Zero-based media line index for the transceiver. This is the index of the + /// m= line in the SDP offer/answer as determined when adding the transceiver. + /// This is provided by the implementation and is immutable (since we don't + /// support stopping transceivers, so m= lines are not recycled). + int mline_index = -1; + + /// Initial desired direction when the transceiver is created. If the + /// transceiver is created by the implementation, this is generally set to the + /// current value on the implementation object, to keep the interop wrapper in + /// sync. + mrsTransceiverDirection initial_desired_direction = + mrsTransceiverDirection::kSendRecv; }; -/// Add a local video track from a custom video source external to the -/// implementation. This allows feeding into WebRTC frames from any source, -/// including generated or synthetic frames, for example for testing. -/// The track source initially starts as capuring. Capture can be stopped with -/// |mrsExternalVideoTrackSourceShutdown|. -/// This returns a handle to a newly allocated object, which must be released -/// once not used anymore with |mrsLocalVideoTrackRemoveRef()|. -MRS_API mrsResult MRS_CALL -mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - PeerConnectionHandle peerHandle, - const char* track_name, - ExternalVideoTrackSourceHandle source_handle, - const LocalVideoTrackFromExternalSourceInitConfig* config, - LocalVideoTrackHandle* track_handle) noexcept; - -/// Remove a local video track from the given peer connection and destroy it. -/// After this call returned, the video track handle is invalid. -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peer_handle, - LocalVideoTrackHandle track_handle) noexcept; - -/// Remove all local video tracks backed by the given video track source from -/// the given peer connection and destroy the video track source. -/// After this call returned, the video track source handle is invalid. -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTracksFromSource( - PeerConnectionHandle peer_handle, - ExternalVideoTrackSourceHandle source_handle) noexcept; - -/// Add a local audio track from a local audio capture device (microphone) to -/// the collection of tracks to send to the remote peer. -MRS_API mrsResult MRS_CALL -mrsPeerConnectionAddLocalAudioTrack(PeerConnectionHandle peerHandle) noexcept; +/// Configuration for creating a new video transceiver interop wrapper. +struct mrsVideoTransceiverConfig { + /// Transceiver name, for pairing with remote peer's one. This is normally + /// extracted from the receiver's stream ID #0 if the transceiver was created + /// for a remote track, or is determined by the user-provided value if created + /// with |AddVideoTransceiver()|. + const char* name{}; + + /// Zero-based media line index for the transceiver. This is the index of the + /// m= line in the SDP offer/answer as determined when adding the transceiver. + /// This is provided by the implementation and is immutable (since we don't + /// support stopping transceivers, so m= lines are not recycled). + int mline_index = -1; + + /// Initial desired direction when the transceiver is created. If the + /// transceiver is created by the implementation, this is generally set to the + /// current value on the implementation object, to keep the interop wrapper in + /// sync. + mrsTransceiverDirection initial_desired_direction = + mrsTransceiverDirection::kSendRecv; +}; + +struct mrsRemoteAudioTrackConfig { + const char* track_name{}; +}; + +struct mrsRemoteVideoTrackConfig { + const char* track_name{}; +}; enum class mrsDataChannelConfigFlags : uint32_t { kOrdered = 0x1, @@ -615,24 +832,10 @@ MRS_API mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( mrsDataChannelCallbacks callbacks, DataChannelHandle* dataChannelHandleOut) noexcept; -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peerHandle, - LocalVideoTrackHandle trackHandle) noexcept; - -MRS_API void MRS_CALL mrsPeerConnectionRemoveLocalAudioTrack( - PeerConnectionHandle peerHandle) noexcept; - MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( PeerConnectionHandle peerHandle, DataChannelHandle dataChannelHandle) noexcept; -MRS_API mrsResult MRS_CALL -mrsPeerConnectionSetLocalAudioTrackEnabled(PeerConnectionHandle peerHandle, - mrsBool enabled) noexcept; - -MRS_API mrsBool MRS_CALL mrsPeerConnectionIsLocalAudioTrackEnabled( - PeerConnectionHandle peerHandle) noexcept; - MRS_API mrsResult MRS_CALL mrsDataChannelSendMessage(DataChannelHandle dataChannelHandle, const void* data, @@ -776,6 +979,7 @@ MRS_API void MRS_CALL mrsMemCpyStride(void* dst, int32_t src_stride, int32_t elem_size, int32_t elem_count) noexcept; + // // Stats extraction. // @@ -893,4 +1097,5 @@ mrsStatsReportGetObjects(mrsStatsReportHandle report_handle, /// Release a stats report. MRS_API mrsResult MRS_CALL mrsStatsReportRemoveRef(mrsStatsReportHandle stats_report); + } // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h index 1dd1f9f5b..1e1eaf0a6 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h @@ -19,6 +19,21 @@ mrsLocalVideoTrackAddRef(LocalVideoTrackHandle handle) noexcept; MRS_API void MRS_CALL mrsLocalVideoTrackRemoveRef(LocalVideoTrackHandle handle) noexcept; +/// Create a new local video track by opening a local video capture device +/// (webcam). +/// [UWP] This must be invoked from another thread than the main UI thread. +MRS_API mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice( + const LocalVideoTrackInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept; + +/// Create a new local video track by using an existing external video source. +MRS_API mrsResult MRS_CALL mrsLocalVideoTrackCreateFromExternalSource( + ExternalVideoTrackSourceHandle source_handle, + const LocalVideoTrackFromExternalSourceInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept; + /// Register a custom callback to be called when the local video track captured /// a frame. The captured frames is passed to the registered callback in I420 /// encoding. diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h index 4f293d9ec..5b5908823 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h @@ -29,4 +29,18 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterIceGatheringStateChangedCallback( mrsPeerConnectionIceGatheringStateChangedCallback callback, void* user_data) noexcept; +/// Create a new audio transceiver attached to the given peer connection. +/// The audio transceiver is initially inactive. +MRS_API mrsResult MRS_CALL +mrsPeerConnectionAddAudioTransceiver(PeerConnectionHandle peer_handle, + const AudioTransceiverInitConfig* config, + AudioTransceiverHandle* handle) noexcept; + +/// Create a new video transceiver attached to the given peer connection. +/// The audio transceiver is initially inactive. +MRS_API mrsResult MRS_CALL +mrsPeerConnectionAddVideoTransceiver(PeerConnectionHandle peer_handle, + const VideoTransceiverInitConfig* config, + VideoTransceiverHandle* handle) noexcept; + } // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp new file mode 100644 index 000000000..c6d716eeb --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/audio_transceiver_interop.h" +#include "media/audio_transceiver.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsAudioTransceiverAddRef(AudioTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL AudioTransceiver object."; + } +} + +void MRS_CALL +mrsAudioTransceiverRemoveRef(AudioTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to remove reference from NULL AudioTransceiver object."; + } +} + +void MRS_CALL mrsAudioTransceiverRegisterStateUpdatedCallback( + AudioTransceiverHandle handle, + mrsAudioTransceiverStateUpdatedCallback callback, + void* user_data) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RegisterStateUpdatedCallback( + Transceiver::StateUpdatedCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL mrsAudioTransceiverSetDirection( + AudioTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept { + if (auto transceiver = static_cast(transceiver_handle)) { + return transceiver->SetDirection(new_direction); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsAudioTransceiverSetLocalTrack(AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle track_handle) noexcept { + if (!transceiver_handle) { + return Result::kInvalidNativeHandle; + } + auto transceiver = static_cast(transceiver_handle); + auto track = static_cast(track_handle); + return transceiver->SetLocalTrack(track); +} + +mrsResult MRS_CALL mrsAudioTransceiverGetLocalTrack( + AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetLocalTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL mrsAudioTransceiverGetRemoteTrack( + AudioTransceiverHandle transceiver_handle, + RemoteAudioTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetRemoteTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h new file mode 100644 index 000000000..d6b769088 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +using mrsAudioTransceiverStateUpdatedCallback = + void(MRS_CALL*)(void* user_data, + mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated_direction, + mrsTransceiverDirection desired_direction); + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTransceiverAddRef(AudioTransceiverHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTransceiverRemoveRef(AudioTransceiverHandle handle) noexcept; + +MRS_API void MRS_CALL mrsAudioTransceiverRegisterStateUpdatedCallback( + AudioTransceiverHandle handle, + mrsAudioTransceiverStateUpdatedCallback callback, + void* user_data) noexcept; + +/// Set the new desired transceiver direction. +MRS_API mrsResult MRS_CALL +mrsAudioTransceiverSetDirection(AudioTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept; + +/// Set the local audio track associated with this transceiver. This new track +/// replaces the existing one, if any. This doesn't require any SDP +/// renegotiation. +MRS_API mrsResult MRS_CALL +mrsAudioTransceiverSetLocalTrack(AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle track_handle) noexcept; + +/// Get the local audio track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsLocalAudioTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsAudioTransceiverGetLocalTrack( + AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle* track_handle_out) noexcept; + +/// Get the remote audio track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsRemoteAudioTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsAudioTransceiverGetRemoteTrack( + AudioTransceiverHandle transceiver_handle, + RemoteAudioTrackHandle* track_handle_out) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp index 2d9782e86..3418c2a34 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp @@ -21,10 +21,20 @@ using namespace Microsoft::MixedReality::WebRTC; /// always valid, never deallocated. std::string_view ObjectTypeToString(ObjectType type) { static_assert((int)ObjectType::kPeerConnection == 0, ""); - static_assert((int)ObjectType::kLocalVideoTrack == 1, ""); - static_assert((int)ObjectType::kExternalVideoTrackSource == 2, ""); + static_assert((int)ObjectType::kLocalAudioTrack == 1, ""); + static_assert((int)ObjectType::kLocalVideoTrack == 2, ""); + static_assert((int)ObjectType::kExternalVideoTrackSource == 3, ""); + static_assert((int)ObjectType::kRemoteAudioTrack == 4, ""); + static_assert((int)ObjectType::kRemoteVideoTrack == 5, ""); + static_assert((int)ObjectType::kDataChannel == 6, ""); + static_assert((int)ObjectType::kAudioTransceiver == 7, ""); + static_assert((int)ObjectType::kVideoTransceiver == 8, ""); constexpr const std::string_view s_types[] = { - "PeerConnection", "LocalVideoTrack", "ExternalVideoTrackSource"}; + "PeerConnection", "LocalAudioTrack", + "LocalVideoTrack", "ExternalVideoTrackSource", + "RemoteAudioTrack", "RemoteVideoTrack", + "DataChannel", "AudioTransceiver", + "VideoTransceiver"}; return s_types[(int)type]; } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp index f3394155a..9816471c2 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp @@ -10,12 +10,16 @@ #include "data_channel.h" #include "external_video_track_source_interop.h" #include "interop/global_factory.h" +#include "interop/local_audio_track_interop.h" #include "interop_api.h" +#include "local_video_track_interop.h" #include "media/external_video_track_source_impl.h" +#include "media/local_audio_track.h" #include "media/local_video_track.h" #include "peer_connection.h" #include "peer_connection_interop.h" #include "sdp_utils.h" +#include "utils.h" using namespace Microsoft::MixedReality::WebRTC; @@ -630,22 +634,51 @@ void MRS_CALL mrsPeerConnectionRegisterRenegotiationNeededCallback( } } -void MRS_CALL mrsPeerConnectionRegisterTrackAddedCallback( +void MRS_CALL mrsPeerConnectionRegisterAudioTrackAddedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackAddedCallback callback, + PeerConnectionAudioTrackAddedCallback callback, void* user_data) noexcept { if (auto peer = static_cast(peerHandle)) { - peer->RegisterTrackAddedCallback(Callback{callback, user_data}); + peer->RegisterAudioTrackAddedCallback( + Callback{ + callback, user_data}); } } -void MRS_CALL mrsPeerConnectionRegisterTrackRemovedCallback( +void MRS_CALL mrsPeerConnectionRegisterAudioTrackRemovedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackRemovedCallback callback, + PeerConnectionAudioTrackRemovedCallback callback, void* user_data) noexcept { if (auto peer = static_cast(peerHandle)) { - peer->RegisterTrackRemovedCallback( - Callback{callback, user_data}); + peer->RegisterAudioTrackRemovedCallback( + Callback{ + callback, user_data}); + } +} + +void MRS_CALL mrsPeerConnectionRegisterVideoTrackAddedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackAddedCallback callback, + void* user_data) noexcept { + if (auto peer = static_cast(peerHandle)) { + peer->RegisterVideoTrackAddedCallback( + Callback{ + callback, user_data}); + } +} + +void MRS_CALL mrsPeerConnectionRegisterVideoTrackRemovedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackRemovedCallback callback, + void* user_data) noexcept { + if (auto peer = static_cast(peerHandle)) { + peer->RegisterVideoTrackRemovedCallback( + Callback{ + callback, user_data}); } } @@ -671,66 +704,68 @@ void MRS_CALL mrsPeerConnectionRegisterDataChannelRemovedCallback( } } -void MRS_CALL mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsI420AVideoFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback{callback, user_data}); +mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( + const LocalAudioTrackInitConfig* config, + const char* track_name, + LocalAudioTrackHandle* track_handle_out) noexcept { + if (IsStringNullOrEmpty(track_name)) { + RTC_LOG(LS_ERROR) << "Invalid empty local audio track name."; + return Result::kInvalidParameter; } -} + if (!track_handle_out) { + RTC_LOG(LS_ERROR) << "Invalid NULL local audio track handle."; + return Result::kInvalidParameter; + } + *track_handle_out = nullptr; -void MRS_CALL mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsArgb32VideoFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback{callback, user_data}); + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kInvalidOperation; } -} -void MRS_CALL mrsPeerConnectionRegisterLocalAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback{callback, user_data}); + // Create the audio track source + rtc::scoped_refptr audio_source = + pc_factory->CreateAudioSource(cricket::AudioOptions()); + if (!audio_source) { + RTC_LOG(LS_ERROR) << "Failed to create local audio source."; + return Result::kUnknownError; } -} -void MRS_CALL mrsPeerConnectionRegisterRemoteAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback{callback, user_data}); + // Create the audio track + rtc::scoped_refptr audio_track = + pc_factory->CreateAudioTrack(track_name, audio_source); + if (!audio_track) { + RTC_LOG(LS_ERROR) << "Failed to create local audio track."; + return Result::kUnknownError; } + + // Create the audio track wrapper + RefPtr track = + new LocalAudioTrack(std::move(global_factory), std::move(audio_track), + config->track_interop_handle); + *track_handle_out = track.release(); + return Result::kSuccess; } -mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( - PeerConnectionHandle peerHandle, - const char* track_name, +mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice( const LocalVideoTrackInitConfig* config, + const char* track_name, LocalVideoTrackHandle* track_handle_out) noexcept { if (IsStringNullOrEmpty(track_name)) { RTC_LOG(LS_ERROR) << "Invalid empty local video track name."; return Result::kInvalidParameter; } + if (!config) { + RTC_LOG(LS_ERROR) << "Invalid NULL local video device configuration."; + return Result::kInvalidParameter; + } if (!track_handle_out) { RTC_LOG(LS_ERROR) << "Invalid NULL local video track handle."; return Result::kInvalidParameter; } *track_handle_out = nullptr; - auto peer = static_cast(peerHandle); - if (!peer) { - RTC_LOG(LS_ERROR) << "Invalid NULL peer connection handle."; - return Result::kInvalidNativeHandle; - } RefPtr global_factory(GlobalFactory::InstancePtr()); auto pc_factory = global_factory->GetPeerConnectionFactory(); if (!pc_factory) { @@ -767,119 +802,30 @@ mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( SimpleMediaConstraints::MaxFrameRate(config->framerate)); } + // Create the video track source rtc::scoped_refptr video_source = pc_factory->CreateVideoSource(std::move(video_capturer), videoConstraints.get()); if (!video_source) { return Result::kUnknownError; } + + // Create the video track rtc::scoped_refptr video_track = pc_factory->CreateVideoTrack(track_name, video_source); if (!video_track) { RTC_LOG(LS_ERROR) << "Failed to create local video track."; return Result::kUnknownError; } - auto result = peer->AddLocalVideoTrack(std::move(video_track), - config->track_interop_handle); - if (result.ok()) { - RefPtr& video_track_wrapper = result.value(); - video_track_wrapper->AddRef(); // for the handle - *track_handle_out = video_track_wrapper.get(); - return Result::kSuccess; - } - RTC_LOG(LS_ERROR) << "Failed to add local video track to peer connection."; - return Result::kUnknownError; -} -mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - PeerConnectionHandle peer_handle, - const char* track_name, - ExternalVideoTrackSourceHandle source_handle, - const LocalVideoTrackFromExternalSourceInitConfig* config, - LocalVideoTrackHandle* track_handle_out) noexcept { - if (!track_handle_out) { - return Result::kInvalidParameter; - } - *track_handle_out = nullptr; - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto track_source = - static_cast(source_handle); - if (!track_source) { - return Result::kInvalidNativeHandle; - } - RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); - if (!pc_factory) { - return Result::kUnknownError; - } - std::string track_name_str; - if (track_name && (track_name[0] != '\0')) { - track_name_str = track_name; - } else { - track_name_str = "external_track"; - } - // The video track keeps a reference to the video source; let's hope this - // does not change, because this is not explicitly mentioned in the docs, - // and the video track is the only one keeping the video source alive. - rtc::scoped_refptr video_track = - pc_factory->CreateVideoTrack(track_name_str, track_source->impl()); - if (!video_track) { - return Result::kUnknownError; - } - auto result = peer->AddLocalVideoTrack(std::move(video_track), - config->track_interop_handle); - if (result.ok()) { - *track_handle_out = result.value().release(); - return Result::kSuccess; - } - RTC_LOG(LS_ERROR) << "Failed to add local video track: " - << result.error().message(); - return Result::kUnknownError; //< TODO Convert from result.error()? -} - -mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTracksFromSource( - PeerConnectionHandle peer_handle, - ExternalVideoTrackSourceHandle source_handle) noexcept { - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto source = static_cast(source_handle); - if (!source) { - return Result::kInvalidNativeHandle; - } - peer->RemoveLocalVideoTracksFromSource(*source); + // Create the video track wrapper + RefPtr track = + new LocalVideoTrack(std::move(global_factory), std::move(video_track), + config->track_interop_handle); + *track_handle_out = track.release(); return Result::kSuccess; } -mrsResult MRS_CALL -mrsPeerConnectionAddLocalAudioTrack(PeerConnectionHandle peerHandle) noexcept { - if (auto peer = static_cast(peerHandle)) { - RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); - if (!pc_factory) { - return Result::kInvalidOperation; - } - rtc::scoped_refptr audio_source = - pc_factory->CreateAudioSource(cricket::AudioOptions()); - if (!audio_source) { - return Result::kUnknownError; - } - rtc::scoped_refptr audio_track = - pc_factory->CreateAudioTrack(kLocalAudioLabel, audio_source); - if (!audio_track) { - return Result::kUnknownError; - } - return (peer->AddLocalAudioTrack(std::move(audio_track)) - ? Result::kSuccess - : Result::kUnknownError); - } - return Result::kUnknownError; -} - mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( PeerConnectionHandle peerHandle, mrsDataChannelInteropHandle dataChannelInteropHandle, @@ -916,30 +862,6 @@ mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( return data_channel.error().result(); } -mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peer_handle, - LocalVideoTrackHandle track_handle) noexcept { - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto track = static_cast(track_handle); - if (!track) { - return Result::kInvalidNativeHandle; - } - const mrsResult res = - (peer->RemoveLocalVideoTrack(*track).ok() ? Result::kSuccess - : Result::kUnknownError); - return res; -} - -void MRS_CALL mrsPeerConnectionRemoveLocalAudioTrack( - PeerConnectionHandle peerHandle) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RemoveLocalAudioTrack(); - } -} - mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( PeerConnectionHandle peerHandle, DataChannelHandle dataChannelHandle) noexcept { @@ -955,26 +877,6 @@ mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( return Result::kSuccess; } -mrsResult MRS_CALL -mrsPeerConnectionSetLocalAudioTrackEnabled(PeerConnectionHandle peerHandle, - mrsBool enabled) noexcept { - auto peer = static_cast(peerHandle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - peer->SetLocalAudioTrackEnabled(enabled != mrsBool::kFalse); - return Result::kSuccess; -} - -mrsBool MRS_CALL mrsPeerConnectionIsLocalAudioTrackEnabled( - PeerConnectionHandle peerHandle) noexcept { - auto peer = static_cast(peerHandle); - if (!peer) { - return mrsBool::kFalse; - } - return (peer->IsLocalAudioTrackEnabled() ? mrsBool::kTrue : mrsBool::kFalse); -} - mrsResult MRS_CALL mrsDataChannelSendMessage(DataChannelHandle dataChannelHandle, const void* data, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp new file mode 100644 index 000000000..f0ea6082b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/local_audio_track_interop.h" +#include "media/local_audio_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL mrsLocalAudioTrackAddRef(LocalAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL LocalAudioTrack object."; + } +} + +void MRS_CALL +mrsLocalAudioTrackRemoveRef(LocalAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "LocalAudioTrack object."; + } +} + +// mrsLocalAudioTrackCreateFromDevice -> interop_api.cpp + +void MRS_CALL +mrsLocalAudioTrackRegisterFrameCallback(LocalAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(AudioFrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsLocalAudioTrackSetEnabled(LocalAudioTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsLocalAudioTrackIsEnabled(LocalAudioTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h new file mode 100644 index 000000000..ac578af78 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsLocalAudioTrackAddRef(LocalAudioTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsLocalAudioTrackRemoveRef(LocalAudioTrackHandle handle) noexcept; + +/// Create a new local audio track by opening a local audio capture device +/// (webcam). +MRS_API mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( + const LocalAudioTrackInitConfig* config, + const char* track_name, + LocalAudioTrackHandle* track_handle_out) noexcept; + +/// Register a custom callback to be called when the local audio track captured +/// a frame. +MRS_API void MRS_CALL +mrsLocalAudioTrackRegisterFrameCallback(LocalAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a local audio track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (silent +/// audio frames). Enabling/disabling a track is a lightweight concept similar +/// to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsLocalAudioTrackSetEnabled(LocalAudioTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local audio track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsLocalAudioTrackIsEnabled(LocalAudioTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp index 95742cd02..56b077f59 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp @@ -5,8 +5,11 @@ // line, to prevent clang-format from reordering it with other headers. #include "pch.h" +#include "global_factory.h" #include "local_video_track_interop.h" +#include "media/external_video_track_source_impl.h" #include "media/local_video_track.h" +#include "utils.h" using namespace Microsoft::MixedReality::WebRTC; @@ -29,6 +32,54 @@ mrsLocalVideoTrackRemoveRef(LocalVideoTrackHandle handle) noexcept { } } +// mrsLocalVideoTrackCreateFromDevice -> interop_api.cpp + +mrsResult MRS_CALL mrsLocalVideoTrackCreateFromExternalSource( + ExternalVideoTrackSourceHandle source_handle, + const LocalVideoTrackFromExternalSourceInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + *track_handle_out = nullptr; + + auto track_source = + static_cast(source_handle); + if (!track_source) { + return Result::kInvalidNativeHandle; + } + + std::string track_name_str; + if (!IsStringNullOrEmpty(track_name)) { + track_name_str = track_name; + } else { + track_name_str = "external_track"; + } + + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kUnknownError; + } + + // The video track keeps a reference to the video source; let's hope this + // does not change, because this is not explicitly mentioned in the docs, + // and the video track is the only one keeping the video source alive. + rtc::scoped_refptr video_track = + pc_factory->CreateVideoTrack(track_name_str, track_source->impl()); + if (!video_track) { + return Result::kUnknownError; + } + + // Create the video track wrapper + RefPtr track = + new LocalVideoTrack(std::move(global_factory), std::move(video_track), + config->track_interop_handle); + *track_handle_out = track.release(); + return Result::kSuccess; +} + void MRS_CALL mrsLocalVideoTrackRegisterI420AFrameCallback( LocalVideoTrackHandle trackHandle, mrsI420AVideoFrameCallback callback, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp index 4deda7fa2..fe3ee4a24 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp @@ -38,3 +38,43 @@ void MRS_CALL mrsPeerConnectionRegisterIceGatheringStateChangedCallback( Callback{callback, user_data}); } } + +mrsResult MRS_CALL +mrsPeerConnectionAddAudioTransceiver(PeerConnectionHandle peer_handle, + const AudioTransceiverInitConfig* config, + AudioTransceiverHandle* handle) noexcept { + if (!handle || !config) { + return Result::kInvalidParameter; + } + *handle = nullptr; + if (auto peer = static_cast(peer_handle)) { + ErrorOr> audio_transceiver = + peer->AddAudioTransceiver(*config); + if (audio_transceiver.ok()) { + *handle = (AudioTransceiverHandle)audio_transceiver.value().release(); + return Result::kSuccess; + } + return audio_transceiver.error().result(); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsPeerConnectionAddVideoTransceiver(PeerConnectionHandle peer_handle, + const VideoTransceiverInitConfig* config, + VideoTransceiverHandle* handle) noexcept { + if (!handle || !config) { + return Result::kInvalidParameter; + } + *handle = nullptr; + if (auto peer = static_cast(peer_handle)) { + ErrorOr> video_transceiver = + peer->AddVideoTransceiver(*config); + if (video_transceiver.ok()) { + *handle = (AudioTransceiverHandle)video_transceiver.value().release(); + return Result::kSuccess; + } + return video_transceiver.error().result(); + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp new file mode 100644 index 000000000..85d090b93 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/remote_audio_track_interop.h" +#include "media/remote_audio_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsRemoteAudioTrackAddRef(RemoteAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL RemoteAudioTrack object."; + } +} + +void MRS_CALL +mrsRemoteAudioTrackRemoveRef(RemoteAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "RemoteAudioTrack object."; + } +} + +void MRS_CALL +mrsRemoteAudioTrackRegisterFrameCallback(RemoteAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(AudioFrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsRemoteAudioTrackSetEnabled(RemoteAudioTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsRemoteAudioTrackIsEnabled(RemoteAudioTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h new file mode 100644 index 000000000..2ce7d20ee --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteAudioTrackAddRef(RemoteAudioTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteAudioTrackRemoveRef(RemoteAudioTrackHandle handle) noexcept; + +/// Register a custom callback to be called when the local audio track received +/// a frame. +MRS_API void MRS_CALL +mrsRemoteAudioTrackRegisterFrameCallback(RemoteAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a local audio track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (silent +/// audio frames). Enabling/disabling a track is a lightweight concept similar +/// to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsRemoteAudioTrackSetEnabled(RemoteAudioTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local audio track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsRemoteAudioTrackIsEnabled(RemoteAudioTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp new file mode 100644 index 000000000..68cf47c2c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/remote_video_track_interop.h" +#include "media/remote_video_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsRemoteVideoTrackAddRef(RemoteVideoTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL RemoteVideoTrack object."; + } +} + +void MRS_CALL +mrsRemoteVideoTrackRemoveRef(RemoteVideoTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "RemoteVideoTrack object."; + } +} + +void MRS_CALL mrsRemoteVideoTrackRegisterI420AFrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsI420AVideoFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(I420AFrameReadyCallback{callback, user_data}); + } +} + +void MRS_CALL mrsRemoteVideoTrackRegisterArgb32FrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsArgb32VideoFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(Argb32FrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsRemoteVideoTrackSetEnabled(RemoteVideoTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsRemoteVideoTrackIsEnabled(RemoteVideoTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h new file mode 100644 index 000000000..21c2e61ec --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteVideoTrackAddRef(RemoteVideoTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteVideoTrackRemoveRef(RemoteVideoTrackHandle handle) noexcept; + +/// Register a custom callback to be called when the remote video track received +/// a frame. The received frames is passed to the registered callback in I420 +/// encoding. +MRS_API void MRS_CALL mrsRemoteVideoTrackRegisterI420AFrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsI420AVideoFrameCallback callback, + void* user_data) noexcept; + +/// Register a custom callback to be called when the remote video track received +/// a frame. The received frames is passed to the registered callback in ARGB32 +/// encoding. +MRS_API void MRS_CALL mrsRemoteVideoTrackRegisterArgb32FrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsArgb32VideoFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a remote video track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (black video +/// frames, silent audio frames). Enabling/disabling a track is a lightweight +/// concept similar to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsRemoteVideoTrackSetEnabled(RemoteVideoTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local video track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsRemoteVideoTrackIsEnabled(RemoteVideoTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp new file mode 100644 index 000000000..dd72d16a2 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/video_transceiver_interop.h" +#include "media/video_transceiver.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsVideoTransceiverAddRef(VideoTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL VideoTransceiver object."; + } +} + +void MRS_CALL +mrsVideoTransceiverRemoveRef(VideoTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to remove reference from NULL VideoTransceiver object."; + } +} + +void MRS_CALL mrsVideoTransceiverRegisterStateUpdatedCallback( + VideoTransceiverHandle handle, + mrsVideoTransceiverStateUpdatedCallback callback, + void* user_data) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RegisterStateUpdatedCallback( + Transceiver::StateUpdatedCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL mrsVideoTransceiverSetDirection( + VideoTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept { + if (auto transceiver = static_cast(transceiver_handle)) { + return transceiver->SetDirection(new_direction); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsVideoTransceiverSetLocalTrack(VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle track_handle) noexcept { + if (!transceiver_handle) { + return Result::kInvalidNativeHandle; + } + auto transceiver = static_cast(transceiver_handle); + auto track = static_cast(track_handle); + return transceiver->SetLocalTrack(track); +} + +mrsResult MRS_CALL mrsVideoTransceiverGetLocalTrack( + VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetLocalTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL mrsVideoTransceiverGetRemoteTrack( + VideoTransceiverHandle transceiver_handle, + RemoteVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetRemoteTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h new file mode 100644 index 000000000..ff0dd6bd6 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +using mrsVideoTransceiverStateUpdatedCallback = + void(MRS_CALL*)(void* user_data, + mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated_direction, + mrsTransceiverDirection desired_direction); + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsVideoTransceiverAddRef(VideoTransceiverHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsVideoTransceiverRemoveRef(VideoTransceiverHandle handle) noexcept; + +MRS_API void MRS_CALL mrsVideoTransceiverRegisterStateUpdatedCallback( + VideoTransceiverHandle handle, + mrsVideoTransceiverStateUpdatedCallback callback, + void* user_data) noexcept; + +/// Set the new desired transceiver direction. +MRS_API mrsResult MRS_CALL +mrsVideoTransceiverSetDirection(VideoTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept; + +/// Set the local video track associated with this transceiver. This new +/// track replaces the existing one, if any. This doesn't require any SDP +/// renegotiation. +MRS_API mrsResult MRS_CALL +mrsVideoTransceiverSetLocalTrack(VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle track_handle) noexcept; + +/// Get the local video track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsLocalVideoTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsVideoTransceiverGetLocalTrack( + VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle* track_handle_out) noexcept; + +/// Get the remote video track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsRemoteVideoTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsVideoTransceiverGetRemoteTrack( + VideoTransceiverHandle transceiver_handle, + RemoteVideoTrackHandle* track_handle_out) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp new file mode 100644 index 000000000..61ba129d7 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "audio_transceiver.h" +#include "interop/global_factory.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +AudioTransceiver::AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsAudioTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), MediaKind::kAudio, owner), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +AudioTransceiver::AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsAudioTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), + MediaKind::kAudio, + owner, + transceiver), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +AudioTransceiver::~AudioTransceiver() { + // Be sure to clean-up WebRTC objects before unregistering ourself, which + // could lead to the GlobalFactory being destroyed and the WebRTC threads + // stopped. + transceiver_ = nullptr; +} + +Result AudioTransceiver::SetDirection(Direction new_direction) noexcept { + if (transceiver_) { // Unified Plan + if (new_direction == desired_direction_) { + return Result::kSuccess; + } + transceiver_->SetDirection(ToRtp(new_direction)); + } else { // Plan B + //< TODO + return Result::kUnknownError; + } + desired_direction_ = new_direction; + FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason::kSetDirection); + return Result::kSuccess; +} + +Result AudioTransceiver::SetLocalTrack( + RefPtr local_track) noexcept { + if (local_track_ == local_track) { + return Result::kSuccess; + } + Result result = Result::kSuccess; + if (transceiver_) { // Unified Plan + // We are running under the assumption that SetTrack() never changes any of + // the transceiver's directions. This is not 100% clear in the standard, so + // double-check it here in Debug, under RTC_DCHECK_IS_ON because it's + // potentially a bit expensive (proxied calls). +#if RTC_DCHECK_IS_ON + auto desired0 = transceiver_->direction(); + auto negotiated0 = transceiver_->current_direction(); +#endif + if (!transceiver_->sender()->SetTrack(local_track ? local_track->impl() + : nullptr)) { + result = Result::kInvalidOperation; + } +#if RTC_DCHECK_IS_ON + auto desired1 = transceiver_->direction(); + auto negotiated1 = transceiver_->current_direction(); + RTC_DCHECK(desired0 == desired1); + RTC_DCHECK(negotiated0 == negotiated1); +#endif + } else { // Plan B + // auto ret = owner_->ReplaceTrackPlanB(local_track_, local_track); + // if (!ret.ok()) { + // result = ret.error().result(); + //} + result = Result::kUnknownError; //< TODO + } + if (result != Result::kSuccess) { + if (local_track) { + RTC_LOG(LS_ERROR) << "Failed to set local audio track " + << local_track->GetName() << " of audio transceiver " + << GetName() << "."; + } else { + RTC_LOG(LS_ERROR) + << "Failed to clear local audio track from audio transceiver " + << GetName() << "."; + } + return result; + } + if (local_track_) { + // Detach old local track + // Keep a pointer to the track because local_track_ gets NULL'd. No need to + // keep a reference, because owner_ has one. + LocalAudioTrack* const track = local_track_.get(); + track->OnRemovedFromPeerConnection(*owner_, this, transceiver_->sender()); + owner_->OnLocalTrackRemovedFromAudioTransceiver(*this, *track); + } + local_track_ = std::move(local_track); + if (local_track_) { + // Attach new local track + local_track_->OnAddedToPeerConnection(*owner_, this, + transceiver_->sender()); + owner_->OnLocalTrackAddedToAudioTransceiver(*this, *local_track_); + } + return Result::kSuccess; +} + +void AudioTransceiver::OnLocalTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!local_track_ || (local_track_ == track)); + local_track_ = std::move(track); +} + +void AudioTransceiver::OnRemoteTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!remote_track_ || (remote_track_ == track)); + remote_track_ = std::move(track); +} + +void AudioTransceiver::OnLocalTrackRemoved(LocalAudioTrack* track) { + RTC_DCHECK_EQ(track, local_track_.get()); + local_track_ = nullptr; +} + +void AudioTransceiver::OnRemoteTrackRemoved(RemoteAudioTrack* track) { + RTC_DCHECK_EQ(track, remote_track_.get()); + remote_track_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h new file mode 100644 index 000000000..c2451228c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "media/local_audio_track.h" +#include "media/remote_audio_track.h" +#include "media/transceiver.h" +#include "refptr.h" + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Transceiver containing audio tracks. +class AudioTransceiver : public Transceiver { + public: + /// Constructor for Plan B. + AudioTransceiver(RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsAudioTransceiverInteropHandle interop_handle) noexcept; + + /// Constructor for Unified Plan. + AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsAudioTransceiverInteropHandle interop_handle) noexcept; + + ~AudioTransceiver() override; + + Result SetDirection(Direction new_direction) noexcept; + + Result SetLocalTrack(RefPtr local_track) noexcept; + + RefPtr GetLocalTrack() noexcept { return local_track_; } + + RefPtr GetRemoteTrack() noexcept { return remote_track_; } + + int GetMlineIndex() const noexcept { return mline_index_; } + + // + // Internal + // + + void OnLocalTrackAdded(RefPtr track); + void OnRemoteTrackAdded(RefPtr track); + void OnLocalTrackRemoved(LocalAudioTrack* track); + void OnRemoteTrackRemoved(RemoteAudioTrack* track); + mrsAudioTransceiverInteropHandle GetInteropHandle() const { + return interop_handle_; + } + + protected: + RefPtr local_track_; + RefPtr remote_track_; + + int mline_index_ = -1; + + /// Transceiver name, for pairing with the remote peer. + std::string name_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsAudioTransceiverInteropHandle interop_handle_{}; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp new file mode 100644 index 000000000..86e4d3cca --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "local_audio_track.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +LocalAudioTrack::LocalAudioTrack( + RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), ObjectType::kLocalAudioTrack), + track_(std::move(track)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(track_); + kind_ = TrackKind::kAudioTrack; + track_->AddSink(this); //< FIXME - Implementation is no-op +} + +LocalAudioTrack::LocalAudioTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr sender, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kLocalAudioTrack, + owner), + track_(std::move(track)), + sender_(std::move(sender)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(transceiver_); + RTC_CHECK(track_); + RTC_CHECK(sender_); + kind_ = TrackKind::kAudioTrack; + transceiver_->OnLocalTrackAdded(this); + track_->AddSink(this); //< FIXME - Implementation is no-op +} + +LocalAudioTrack::~LocalAudioTrack() { + track_->RemoveSink(this); + if (owner_) { + owner_->RemoveLocalAudioTrack(*this); + } + RTC_CHECK(!transceiver_); + RTC_CHECK(!owner_); +} + +void LocalAudioTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); +} + +bool LocalAudioTrack::IsEnabled() const noexcept { + return track_->enabled(); +} + +RefPtr LocalAudioTrack::GetTransceiver() const noexcept { + return transceiver_; +} + +webrtc::AudioTrackInterface* LocalAudioTrack::impl() const { + return track_.get(); +} + +webrtc::RtpSenderInterface* LocalAudioTrack::sender() const { + return sender_.get(); +} + +void LocalAudioTrack::OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender) { + RTC_CHECK(!owner_); + RTC_CHECK(!transceiver_); + RTC_CHECK(!sender_); + RTC_CHECK(transceiver); + RTC_CHECK(sender); + owner_ = &owner; + sender_ = std::move(sender); + transceiver_ = std::move(transceiver); + transceiver_->OnLocalTrackAdded(this); +} + +void LocalAudioTrack::OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender) { + RTC_CHECK_EQ(owner_, &old_owner); + RTC_CHECK_EQ(transceiver_.get(), old_transceiver.get()); + RTC_CHECK_EQ(sender_.get(), old_sender.get()); + owner_ = nullptr; + sender_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; +} + +void LocalAudioTrack::RemoveFromPeerConnection( + webrtc::PeerConnectionInterface& peer) { + if (sender_) { + peer.RemoveTrack(sender_); + sender_ = nullptr; + owner_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; + } +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h new file mode 100644 index 000000000..b0b9e66d8 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "callback.h" +#include "interop_api.h" +#include "media/media_track.h" +#include "refptr.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpSenderInterface; +class AudioTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class AudioTransceiver; + +/// A local audio track is a media track for a peer connection backed by a local +/// source, and transmitted to a remote peer. +/// +/// The local nature of the track implies that the local peer has control on it, +/// including enabling or disabling the track, and removing it from the peer +/// connection. This is in contrast with a remote track reflecting a track sent +/// by the remote peer, for which the local peer has limited control. +/// +/// The local audio track is backed by a local audio track source. This is +/// typically a audio capture device (e.g. webcam), but can also be a source +/// producing programmatically generated frames. The local audio track itself +/// has no knowledge about how the source produces the frames. +class LocalAudioTrack : public AudioFrameObserver, public MediaTrack { + public: + /// Constructor for a track not added to any peer connection. + LocalAudioTrack(RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept; + + /// Constructor for a track added to a peer connection. + LocalAudioTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr sender, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept; + + ~LocalAudioTrack() override; + + /// Get the name of the local audio track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the audio track. An enabled track streams its content + /// from its source to the remote peer. A disabled audio track only sends + /// black frames. + void SetEnabled(bool enabled) const noexcept; + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept; + + [[nodiscard]] RefPtr GetTransceiver() const noexcept; + + // + // Advanced use + // + + [[nodiscard]] webrtc::AudioTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpSenderInterface* sender() const; + + [[nodiscard]] mrsLocalAudioTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + /// Internal callback on added to a peer connection to update the internal + /// state of the object. + void OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender); + + /// Internal callback on removed from a peer connection to update the internal + /// state of the object. + void OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender); + + void RemoveFromPeerConnection(webrtc::PeerConnectionInterface& peer); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr sender_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsLocalAudioTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp index 559308fec..2a0f467c1 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp @@ -9,18 +9,42 @@ namespace Microsoft::MixedReality::WebRTC { +LocalVideoTrack::LocalVideoTrack( + RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalVideoTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), ObjectType::kLocalVideoTrack), + track_(std::move(track)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(track_); + kind_ = TrackKind::kVideoTrack; + rtc::VideoSinkWants sink_settings{}; + sink_settings.rotation_applied = true; + track_->AddOrUpdateSink(this, sink_settings); +} + LocalVideoTrack::LocalVideoTrack( RefPtr global_factory, PeerConnection& owner, + RefPtr transceiver, rtc::scoped_refptr track, rtc::scoped_refptr sender, mrsLocalVideoTrackInteropHandle interop_handle) noexcept - : TrackedObject(std::move(global_factory), ObjectType::kLocalVideoTrack), - owner_(&owner), + : MediaTrack(std::move(global_factory), + ObjectType::kLocalVideoTrack, + owner), track_(std::move(track)), sender_(std::move(sender)), - interop_handle_(interop_handle) { + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { RTC_CHECK(owner_); + RTC_CHECK(transceiver_); + RTC_CHECK(track_); + RTC_CHECK(sender_); + kind_ = TrackKind::kVideoTrack; + transceiver_->OnLocalTrackAdded(this); rtc::VideoSinkWants sink_settings{}; sink_settings.rotation_applied = true; track_->AddOrUpdateSink(this, sink_settings); @@ -31,19 +55,20 @@ LocalVideoTrack::~LocalVideoTrack() { if (owner_) { owner_->RemoveLocalVideoTrack(*this); } + RTC_CHECK(!transceiver_); RTC_CHECK(!owner_); } -std::string LocalVideoTrack::GetName() const noexcept { - return track_->id(); +void LocalVideoTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); } bool LocalVideoTrack::IsEnabled() const noexcept { return track_->enabled(); } -void LocalVideoTrack::SetEnabled(bool enabled) const noexcept { - track_->set_enabled(enabled); +RefPtr LocalVideoTrack::GetTransceiver() const noexcept { + return transceiver_; } webrtc::VideoTrackInterface* LocalVideoTrack::impl() const { @@ -54,12 +79,42 @@ webrtc::RtpSenderInterface* LocalVideoTrack::sender() const { return sender_.get(); } +void LocalVideoTrack::OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender) { + RTC_CHECK(!owner_); + RTC_CHECK(!transceiver_); + RTC_CHECK(!sender_); + RTC_CHECK(transceiver); + RTC_CHECK(sender); + owner_ = &owner; + sender_ = std::move(sender); + transceiver_ = std::move(transceiver); + transceiver_->OnLocalTrackAdded(this); +} + +void LocalVideoTrack::OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender) { + RTC_CHECK_EQ(owner_, &old_owner); + RTC_CHECK_EQ(transceiver_.get(), old_transceiver.get()); + RTC_CHECK_EQ(sender_.get(), old_sender.get()); + owner_ = nullptr; + sender_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; +} + void LocalVideoTrack::RemoveFromPeerConnection( webrtc::PeerConnectionInterface& peer) { if (sender_) { peer.RemoveTrack(sender_); sender_ = nullptr; owner_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; } } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h index 9966b5ac7..80de95cf3 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h @@ -5,6 +5,8 @@ #include "callback.h" #include "interop_api.h" +#include "media/media_track.h" +#include "refptr.h" #include "tracked_object.h" #include "video_frame_observer.h" @@ -21,6 +23,7 @@ class VideoTrackInterface; namespace Microsoft::MixedReality::WebRTC { class PeerConnection; +class VideoTransceiver; /// A local video track is a media track for a peer connection backed by a local /// source, and transmitted to a remote peer. @@ -34,17 +37,25 @@ class PeerConnection; /// typically a video capture device (e.g. webcam), but can also be a source /// producing programmatically generated frames. The local video track itself /// has no knowledge about how the source produces the frames. -class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { +class LocalVideoTrack : public VideoFrameObserver, public MediaTrack { public: + /// Constructor for a track not added to any peer connection. + LocalVideoTrack(RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalVideoTrackInteropHandle interop_handle) noexcept; + + /// Constructor for a track added to a peer connection. LocalVideoTrack(RefPtr global_factory, PeerConnection& owner, + RefPtr transceiver, rtc::scoped_refptr track, rtc::scoped_refptr sender, mrsLocalVideoTrackInteropHandle interop_handle) noexcept; + ~LocalVideoTrack() override; /// Get the name of the local video track. - std::string GetName() const noexcept override; + std::string GetName() const noexcept override { return track_name_; } /// Enable or disable the video track. An enabled track streams its content /// from its source to the remote peer. A disabled video track only sends @@ -55,6 +66,8 @@ class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { /// See |SetEnabled(bool)|. [[nodiscard]] bool IsEnabled() const noexcept; + [[nodiscard]] RefPtr GetTransceiver() const noexcept; + // // Advanced use // @@ -67,20 +80,37 @@ class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { return interop_handle_; } + /// Internal callback on added to a peer connection to update the internal + /// state of the object. + void OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender); + + /// Internal callback on removed from a peer connection to update the internal + /// state of the object. + void OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender); + void RemoveFromPeerConnection(webrtc::PeerConnectionInterface& peer); private: - /// Weak reference to the PeerConnection object owning this track. - PeerConnection* owner_{}; - /// Underlying core implementation. rtc::scoped_refptr track_; /// RTP sender this track is associated with. rtc::scoped_refptr sender_; + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + /// Optional interop handle, if associated with an interop wrapper. mrsLocalVideoTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; }; } // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp new file mode 100644 index 000000000..f29c7b246 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "media_track.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +MediaTrack::MediaTrack(RefPtr global_factory, + ObjectType object_type) noexcept + : TrackedObject(std::move(global_factory), object_type) {} + +MediaTrack::MediaTrack(RefPtr global_factory, + ObjectType object_type, + PeerConnection& owner) noexcept + : TrackedObject(std::move(global_factory), object_type), owner_(&owner) { + RTC_CHECK(owner_); +} + +MediaTrack::~MediaTrack() { + RTC_CHECK(!owner_); +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h new file mode 100644 index 000000000..aba26aa66 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "str.h" +#include "tracked_object.h" +#include "video_frame_observer.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpSenderInterface; +class VideoTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Base class for all audio and video tracks. +class MediaTrack : public TrackedObject { + public: + MediaTrack(RefPtr global_factory, + ObjectType object_type) noexcept; + MediaTrack(RefPtr global_factory, + ObjectType object_type, + PeerConnection& owner) noexcept; + ~MediaTrack() override; + + /// Get the kind of track. + TrackKind GetKind() const noexcept { return kind_; } + + protected: + /// Weak reference to the PeerConnection object owning this track. + PeerConnection* owner_{}; + + /// Track kind. + TrackKind kind_ = TrackKind::kUnknownTrack; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp new file mode 100644 index 000000000..d265f334b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "remote_audio_track.h" + +namespace Microsoft::MixedReality::WebRTC { + +RemoteAudioTrack::RemoteAudioTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kRemoteAudioTrack, + owner), + track_(std::move(track)), + receiver_(std::move(receiver)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(track_); + RTC_CHECK(receiver_); + RTC_CHECK(transceiver_); + kind_ = TrackKind::kAudioTrack; + transceiver_->OnRemoteTrackAdded(this); + track_->AddSink(this); +} + +RemoteAudioTrack::~RemoteAudioTrack() { + track_->RemoveSink(this); + RTC_CHECK(!owner_); +} + +webrtc::AudioTrackInterface* RemoteAudioTrack::impl() const { + return track_.get(); +} + +webrtc::RtpReceiverInterface* RemoteAudioTrack::receiver() const { + return receiver_.get(); +} + +AudioTransceiver* RemoteAudioTrack::GetTransceiver() const { + return transceiver_.get(); +} + +void RemoteAudioTrack::OnTrackRemoved(PeerConnection& owner) { + RTC_DCHECK(owner_ == &owner); + RTC_DCHECK(receiver_ != nullptr); + RTC_DCHECK(transceiver_.get() != nullptr); + owner_ = nullptr; + receiver_ = nullptr; + transceiver_->OnRemoteTrackRemoved(this); + transceiver_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h new file mode 100644 index 000000000..6ad4567bb --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "callback.h" +#include "interop_api.h" +#include "media_track.h" +#include "refptr.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpReceiverInterface; +class AudioTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class AudioTransceiver; + +/// A remote audio track is a media track for a peer connection backed by a +/// remote audio stream received from the remote peer. +/// +/// The remote nature of the track implies that the remote peer has control on +/// it, including enabling or disabling the track, and removing it from the peer +/// connection. The local peer only has limited control over the track. +class RemoteAudioTrack : public AudioFrameObserver, public MediaTrack { + public: + RemoteAudioTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteAudioTrackInteropHandle interop_handle) noexcept; + ~RemoteAudioTrack() override; + + /// Get the name of the remote audio track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the audio track. An enabled track streams its content + /// from its source to the remote peer. A disabled audio track only sends + /// empty audio data (silence). + void SetEnabled(bool enabled) const noexcept { track_->set_enabled(enabled); } + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept { return track_->enabled(); } + + // + // Advanced use + // + + [[nodiscard]] webrtc::AudioTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpReceiverInterface* receiver() const; + [[nodiscard]] AudioTransceiver* GetTransceiver() const; + + [[nodiscard]] mrsRemoteAudioTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + // Automatically called - do not use. + void OnTrackRemoved(PeerConnection& owner); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr receiver_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsRemoteAudioTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp new file mode 100644 index 000000000..b71e64c2b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "remote_video_track.h" + +namespace Microsoft::MixedReality::WebRTC { + +RemoteVideoTrack::RemoteVideoTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteVideoTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kRemoteVideoTrack, + owner), + track_(std::move(track)), + receiver_(std::move(receiver)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(track_); + RTC_CHECK(receiver_); + RTC_CHECK(transceiver_); + kind_ = TrackKind::kVideoTrack; + transceiver_->OnRemoteTrackAdded(this); + rtc::VideoSinkWants sink_settings{}; + sink_settings.rotation_applied = true; + track_->AddOrUpdateSink(this, sink_settings); +} + +RemoteVideoTrack::~RemoteVideoTrack() { + track_->RemoveSink(this); + RTC_CHECK(!owner_); +} + +bool RemoteVideoTrack::IsEnabled() const noexcept { + return track_->enabled(); +} + +void RemoteVideoTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); +} + +webrtc::VideoTrackInterface* RemoteVideoTrack::impl() const { + return track_.get(); +} + +webrtc::RtpReceiverInterface* RemoteVideoTrack::receiver() const { + return receiver_.get(); +} + +VideoTransceiver* RemoteVideoTrack::GetTransceiver() const { + return transceiver_.get(); +} + +void RemoteVideoTrack::OnTrackRemoved(PeerConnection& owner) { + RTC_DCHECK(owner_ == &owner); + RTC_DCHECK(receiver_ != nullptr); + RTC_DCHECK(transceiver_.get() != nullptr); + owner_ = nullptr; + receiver_ = nullptr; + transceiver_->OnRemoteTrackRemoved(this); + transceiver_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h new file mode 100644 index 000000000..5f4c57eb1 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "media_track.h" +#include "refptr.h" +#include "tracked_object.h" +#include "video_frame_observer.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpReceiverInterface; +class VideoTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class VideoTransceiver; + +/// A remote video track is a media track for a peer connection backed by a +/// remote video stream received from the remote peer. +/// +/// The remote nature of the track implies that the remote peer has control on +/// it, including enabling or disabling the track, and removing it from the peer +/// connection. The local peer only has limited control over the track. +class RemoteVideoTrack : public VideoFrameObserver, public MediaTrack { + public: + RemoteVideoTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteVideoTrackInteropHandle interop_handle) noexcept; + ~RemoteVideoTrack() override; + + /// Get the name of the remote video track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the video track. An enabled track streams its content + /// from its source to the remote peer. A disabled video track only sends + /// black frames. + void SetEnabled(bool enabled) const noexcept; + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept; + + // + // Advanced use + // + + [[nodiscard]] webrtc::VideoTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpReceiverInterface* receiver() const; + [[nodiscard]] VideoTransceiver* GetTransceiver() const; + + [[nodiscard]] mrsRemoteVideoTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + // Automatically called - do not use. + void OnTrackRemoved(PeerConnection& owner); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr receiver_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsRemoteVideoTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp new file mode 100644 index 000000000..a8465aaa0 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "transceiver.h" +#include "utils.h" + +namespace Microsoft::MixedReality::WebRTC { + +Transceiver::Transceiver(RefPtr global_factory, + MediaKind kind, + PeerConnection& owner) noexcept + : TrackedObject(std::move(global_factory), + kind == MediaKind::kAudio ? ObjectType::kAudioTransceiver + : ObjectType::kVideoTransceiver), + owner_(&owner), + kind_(kind) { + RTC_CHECK(owner_); + //< TODO + // RTC_CHECK(owner.sdp_semantic == webrtc::SdpSemantics::kPlanB); +} + +Transceiver::Transceiver( + RefPtr global_factory, + MediaKind kind, + PeerConnection& owner, + rtc::scoped_refptr transceiver) noexcept + : TrackedObject(std::move(global_factory), + kind == MediaKind::kAudio ? ObjectType::kAudioTransceiver + : ObjectType::kVideoTransceiver), + owner_(&owner), + kind_(kind), + transceiver_(std::move(transceiver)) { + RTC_CHECK(owner_); + //< TODO + // RTC_CHECK(owner.sdp_semantic == webrtc::SdpSemantics::kUnifiedPlan); +} + +Transceiver::~Transceiver() { + // RTC_CHECK(!owner_); +} + +std::string Transceiver::GetName() const { + return "TODO"; +} + +rtc::scoped_refptr Transceiver::impl() const { + return transceiver_; +} + +webrtc::RtpTransceiverDirection Transceiver::ToRtp(Direction direction) { + using RtpDir = webrtc::RtpTransceiverDirection; + static_assert((int)Direction::kSendRecv == (int)RtpDir::kSendRecv, ""); + static_assert((int)Direction::kSendOnly == (int)RtpDir::kSendOnly, ""); + static_assert((int)Direction::kRecvOnly == (int)RtpDir::kRecvOnly, ""); + static_assert((int)Direction::kInactive == (int)RtpDir::kInactive, ""); + return (RtpDir)direction; +} + +Transceiver::Direction Transceiver::FromRtp( + webrtc::RtpTransceiverDirection rtp_direction) { + using RtpDir = webrtc::RtpTransceiverDirection; + static_assert((int)Direction::kSendRecv == (int)RtpDir::kSendRecv, ""); + static_assert((int)Direction::kSendOnly == (int)RtpDir::kSendOnly, ""); + static_assert((int)Direction::kRecvOnly == (int)RtpDir::kRecvOnly, ""); + static_assert((int)Direction::kInactive == (int)RtpDir::kInactive, ""); + return (Direction)rtp_direction; +} + +Transceiver::OptDirection Transceiver::FromRtp( + std::optional rtp_direction) { + if (rtp_direction.has_value()) { + return (OptDirection)FromRtp(rtp_direction.value()); + } + return OptDirection::kNotSet; +} + +std::vector Transceiver::DecodeStreamIDs( + const char* encoded_stream_ids) { + if (IsStringNullOrEmpty(encoded_stream_ids)) { + return {}; + } + std::vector ids; + rtc::split(encoded_stream_ids, ';', &ids); + return ids; +} + +std::string Transceiver::EncodeStreamIDs( + const std::vector& stream_ids) { + if (stream_ids.empty()) { + return {}; + } + return rtc::join(stream_ids, ';'); +} + +void Transceiver::OnSessionDescUpdated(bool remote) { + // Parse state to check for changes + bool changed = false; + if (transceiver_) { // Unified Plan + // Check negotiated direction + auto negotiated = transceiver_->current_direction(); + if (negotiated.has_value()) { + auto newValue = FromRtp(negotiated); + if (newValue != direction_) { + direction_ = newValue; + changed = true; + } + } + + // Check desired direciton + auto desired = transceiver_->direction(); + { + auto newValue = FromRtp(desired); + if (newValue != desired_direction_) { + desired_direction_ = newValue; + changed = true; + } + } + } else { + assert(false); // not called in Plan B for now... + } + + // Invoke interop callback if any + if (changed) { + FireStateUpdatedEvent(remote + ? mrsTransceiverStateUpdatedReason::kRemoteDesc + : mrsTransceiverStateUpdatedReason::kLocalDesc); + } +} + +void Transceiver::FireStateUpdatedEvent( + mrsTransceiverStateUpdatedReason reason) { + auto lock = std::scoped_lock{cb_mutex_}; + if (auto cb = state_updated_callback_) { + cb(reason, direction_, desired_direction_); + } +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h new file mode 100644 index 000000000..474342857 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpTransceiverInterface; +} + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Media kind for tracks and transceivers. +enum class MediaKind : uint32_t { kAudio, kVideo }; + +/// Base class for audio and video transceivers. +class Transceiver : public TrackedObject { + public: + using Direction = mrsTransceiverDirection; + using OptDirection = mrsTransceiverOptDirection; + + /// Construct a Plan B transceiver abstraction which tries to mimic a + /// transceiver for Plan B despite the fact that this semantic doesn't have + /// any concept of transceiver. + Transceiver(RefPtr global_factory, + MediaKind kind, + PeerConnection& owner) noexcept; + + /// Construct a Unified Plan transceiver wrapper referencing an actual WebRTC + /// transceiver implementation object as defined in Unified Plan. + Transceiver( + RefPtr global_factory, + MediaKind kind, + PeerConnection& owner, + rtc::scoped_refptr transceiver) noexcept; + + ~Transceiver() override; + + std::string GetName() const override; + + /// Get the kind of transceiver. This is generally used for determining what + /// type to static_cast<> a |Transceiver| pointer to. If this is |kAudio| then + /// the object is an |AudioTransceiver| instance, and if this is |kVideo| then + /// the object is a |VideoTransceiver| instance. + MediaKind GetMediaKind() const noexcept { return kind_; } + + /// Get the current transceiver direction. + OptDirection GetDirection() const noexcept { return direction_; } + + // + // Interop callbacks + // + + /// Callback invoked when the internal state of the transceiver has + /// been updated. + using StateUpdatedCallback = + Callback; + + void RegisterStateUpdatedCallback(StateUpdatedCallback&& callback) noexcept { + auto lock = std::scoped_lock{cb_mutex_}; + state_updated_callback_ = std::move(callback); + } + + // + // Advanced + // + + [[nodiscard]] rtc::scoped_refptr impl() + const; + + /// Callback on local description updated, to check for any change in the + /// transceiver direction and update its state. + void OnSessionDescUpdated(bool remote); + + /// Fire the StateUpdated event, invoking the |state_updated_callback_| if + /// any is registered. + void FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason reason); + + [[nodiscard]] static webrtc::RtpTransceiverDirection ToRtp( + Direction direction); + [[nodiscard]] static Direction FromRtp( + webrtc::RtpTransceiverDirection rtp_direction); + [[nodiscard]] static OptDirection FromRtp( + std::optional rtp_direction); + + [[nodiscard]] static std::vector DecodeStreamIDs( + const char* encoded_stream_ids); + [[nodiscard]] static std::string EncodeStreamIDs( + const std::vector& stream_ids); + + protected: + /// Weak reference to the PeerConnection object owning this transceiver. + PeerConnection* owner_{}; + + /// Transceiver media kind. + MediaKind kind_; + + /// Current transceiver direction, as last negotiated. + /// This does not map 1:1 with the presence/absence of the local and remote + /// tracks in |AudioTransceiver| and |VideoTransceiver|, which represent the + /// state for the next negotiation, and will differ after changing tracks but + /// before renegotiating them. This does map however with the internal concept + /// of "preferred direction" |webrtc::RtpTransceiverInterface::direction()|. + OptDirection direction_ = OptDirection::kNotSet; + + /// Next desired direction, as set by user via |SetDirection()|. + Direction desired_direction_ = Direction::kInactive; + + /// If the owner PeerConnection uses Unified Plan, pointer to the actual + /// transceiver implementation object. Otherwise NULL for Plan B. + /// This is also used as a cache of which Plan is in use, to avoid querying + /// the peer connection. + rtc::scoped_refptr transceiver_; + + /// Interop callback invoked when the internal state of the transceiver has + /// been updated. + StateUpdatedCallback state_updated_callback_ RTC_GUARDED_BY(cb_mutex_); + + std::mutex cb_mutex_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp new file mode 100644 index 000000000..a5e31bbe8 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "video_transceiver.h" + +namespace Microsoft::MixedReality::WebRTC { + +VideoTransceiver::VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsVideoTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), MediaKind::kVideo, owner), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +VideoTransceiver::VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsVideoTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), + MediaKind::kVideo, + owner, + transceiver), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +VideoTransceiver::~VideoTransceiver() { + // Be sure to clean-up WebRTC objects before unregistering ourself, which + // could lead to the GlobalFactory being destroyed and the WebRTC threads + // stopped. + transceiver_ = nullptr; +} + +Result VideoTransceiver::SetDirection(Direction new_direction) noexcept { + if (transceiver_) { // Unified Plan + if (new_direction == desired_direction_) { + return Result::kSuccess; + } + transceiver_->SetDirection(ToRtp(new_direction)); + } else { // Plan B + //< TODO + return Result::kUnknownError; + } + desired_direction_ = new_direction; + FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason::kSetDirection); + return Result::kSuccess; +} + +Result VideoTransceiver::SetLocalTrack( + RefPtr local_track) noexcept { + if (local_track_ == local_track) { + return Result::kSuccess; + } + Result result = Result::kSuccess; + if (transceiver_) { // Unified Plan + // We are running under the assumption that SetTrack() never changes any of + // the transceiver's directions. This is not 100% clear in the standard, so + // double-check it here in Debug, under RTC_DCHECK_IS_ON because it's + // potentially a bit expensive (proxied calls). +#if RTC_DCHECK_IS_ON + auto desired0 = transceiver_->direction(); + auto negotiated0 = transceiver_->current_direction(); +#endif + if (!transceiver_->sender()->SetTrack(local_track ? local_track->impl() + : nullptr)) { + result = Result::kInvalidOperation; + } +#if RTC_DCHECK_IS_ON + auto desired1 = transceiver_->direction(); + auto negotiated1 = transceiver_->current_direction(); + RTC_DCHECK(desired0 == desired1); + RTC_DCHECK(negotiated0 == negotiated1); +#endif + } else { // Plan B + // auto ret = owner_->ReplaceTrackPlanB(local_track_, local_track); + // if (!ret.ok()) { + // result = ret.error().result(); + //} + result = Result::kUnknownError; //< TODO + } + if (result != Result::kSuccess) { + if (local_track) { + RTC_LOG(LS_ERROR) << "Failed to set local video track " + << local_track->GetName() << " of video transceiver " + << GetName() << "."; + } else { + RTC_LOG(LS_ERROR) + << "Failed to clear local video track from video transceiver " + << GetName() << "."; + } + return result; + } + if (local_track_) { + // Detach old local track + // Keep a pointer to the track because local_track_ gets NULL'd. No need to + // keep a reference, because owner_ has one. + LocalVideoTrack* const track = local_track_.get(); + track->OnRemovedFromPeerConnection(*owner_, this, transceiver_->sender()); + owner_->OnLocalTrackRemovedFromVideoTransceiver(*this, *track); + } + local_track_ = std::move(local_track); + if (local_track_) { + // Attach new local track + local_track_->OnAddedToPeerConnection(*owner_, this, + transceiver_->sender()); + owner_->OnLocalTrackAddedToVideoTransceiver(*this, *local_track_); + } + return Result::kSuccess; +} + +void VideoTransceiver::OnLocalTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!local_track_ || (local_track_ == track)); + local_track_ = std::move(track); +} + +void VideoTransceiver::OnRemoteTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!remote_track_ || (remote_track_ == track)); + remote_track_ = std::move(track); +} + +void VideoTransceiver::OnLocalTrackRemoved(LocalVideoTrack* track) { + RTC_DCHECK_EQ(track, local_track_.get()); + local_track_ = nullptr; +} + +void VideoTransceiver::OnRemoteTrackRemoved(RemoteVideoTrack* track) { + RTC_DCHECK_EQ(track, remote_track_.get()); + remote_track_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h new file mode 100644 index 000000000..d21011f48 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "local_video_track.h" +#include "media/remote_video_track.h" +#include "media/transceiver.h" +#include "refptr.h" + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Transceiver containing video tracks. +class VideoTransceiver : public Transceiver { + public: + /// Constructor for Plan B. + VideoTransceiver(RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsVideoTransceiverInteropHandle interop_handle) noexcept; + + /// Constructor for Unified Plan. + VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsVideoTransceiverInteropHandle interop_handle) noexcept; + + ~VideoTransceiver() override; + + Result SetDirection(Direction new_direction) noexcept; + + Result SetLocalTrack(RefPtr local_track) noexcept; + + RefPtr GetLocalTrack() noexcept { return local_track_; } + + RefPtr GetRemoteTrack() noexcept { return remote_track_; } + + int GetMlineIndex() const noexcept { return mline_index_; } + + // + // Internal + // + + void OnLocalTrackAdded(RefPtr track); + void OnRemoteTrackAdded(RefPtr track); + void OnLocalTrackRemoved(LocalVideoTrack* track); + void OnRemoteTrackRemoved(RemoteVideoTrack* track); + mrsVideoTransceiverInteropHandle GetInteropHandle() const { + return interop_handle_; + } + + protected: + RefPtr local_track_; + RefPtr remote_track_; + + int mline_index_ = -1; + + /// Transceiver name, for pairing with the remote peer. + std::string name_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsVideoTransceiverInteropHandle interop_handle_{}; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp index b61cc53ba..6ee7cefc0 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp @@ -7,15 +7,17 @@ #include "audio_frame_observer.h" #include "data_channel.h" +#include "interop/global_factory.h" +#include "interop_api.h" +#include "media/local_audio_track.h" #include "media/local_video_track.h" +#include "media/remote_audio_track.h" +#include "media/remote_video_track.h" #include "peer_connection.h" #include "sdp_utils.h" +#include "utils.h" #include "video_frame_observer.h" -// Internal -#include "interop/global_factory.h" -#include "interop_api.h" - #include #if defined(_M_IX86) /* x86 */ && defined(WINAPI_FAMILY) && \ @@ -146,6 +148,25 @@ Microsoft::MixedReality::WebRTC::Error ErrorFromRTCError( ResultFromRTCErrorType(error.type()), error.message()); } +class PeerConnectionImpl; + +class StreamObserver : public webrtc::ObserverInterface { + public: + StreamObserver(PeerConnectionImpl& owner, + rtc::scoped_refptr stream) + : owner_(owner), stream_(std::move(stream)) {} + + protected: + PeerConnectionImpl& owner_; + rtc::scoped_refptr stream_; + + // + // ObserverInterface + // + + void OnChanged() override; +}; + /// Implementation of PeerConnection, which also implements /// PeerConnectionObserver at the same time to simplify interaction with /// the underlying implementation object. @@ -163,9 +184,6 @@ class PeerConnectionImpl : public PeerConnection, void SetPeerImpl(rtc::scoped_refptr impl) { peer_ = std::move(impl); - remote_video_observer_.reset(new VideoFrameObserver()); - local_audio_observer_.reset(new AudioFrameObserver()); - remote_audio_observer_.reset(new AudioFrameObserver()); } void SetName(std::string_view name) { name_ = name; } @@ -228,59 +246,47 @@ class PeerConnectionImpl : public PeerConnection, void Close() noexcept override; bool IsClosed() const noexcept override; - void RegisterTrackAddedCallback( - TrackAddedCallback&& callback) noexcept override { - auto lock = std::scoped_lock{track_added_callback_mutex_}; - track_added_callback_ = std::move(callback); - } - - void RegisterTrackRemovedCallback( - TrackRemovedCallback&& callback) noexcept override { - auto lock = std::scoped_lock{track_removed_callback_mutex_}; - track_removed_callback_ = std::move(callback); - } - - void RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback callback) noexcept override { - if (remote_video_observer_) { - remote_video_observer_->SetCallback(std::move(callback)); - } - } - - void RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback callback) noexcept override { - if (remote_video_observer_) { - remote_video_observer_->SetCallback(std::move(callback)); - } - } - + ErrorOr> AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept override; ErrorOr> AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept override; - webrtc::RTCError RemoveLocalVideoTrack( - LocalVideoTrack& video_track) noexcept override; + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept override; + Result RemoveLocalVideoTrack(LocalVideoTrack& video_track) noexcept override; void RemoveLocalVideoTracksFromSource( ExternalVideoTrackSource& source) noexcept override; - void RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept override { - if (local_audio_observer_) { - local_audio_observer_->SetCallback(std::move(callback)); - } + void RegisterVideoTrackAddedCallback( + VideoTrackAddedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + video_track_added_callback_ = std::move(callback); } - void RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept override { - if (remote_audio_observer_) { - remote_audio_observer_->SetCallback(std::move(callback)); - } + void RegisterVideoTrackRemovedCallback( + VideoTrackRemovedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + video_track_removed_callback_ = std::move(callback); + } + + ErrorOr> AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept override; + ErrorOr> AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept override; + Result RemoveLocalAudioTrack(LocalAudioTrack& audio_track) noexcept override; + + void RegisterAudioTrackAddedCallback( + AudioTrackAddedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + audio_track_added_callback_ = std::move(callback); } - bool AddLocalAudioTrack(rtc::scoped_refptr - audio_track) noexcept override; - void RemoveLocalAudioTrack() noexcept override; - void SetLocalAudioTrackEnabled(bool enabled = true) noexcept override; - bool IsLocalAudioTrackEnabled() const noexcept override; + void RegisterAudioTrackRemovedCallback( + AudioTrackRemovedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + audio_track_removed_callback_ = std::move(callback); + } void RegisterDataChannelAddedCallback( DataChannelAddedCallback callback) noexcept override { @@ -306,11 +312,15 @@ class PeerConnectionImpl : public PeerConnection, mrsResult RegisterInteropCallbacks( const mrsPeerConnectionInteropCallbacks& callbacks) noexcept override { - // Make a full copy of all callbacks + // Make a full copy of all callbacks. Some entries might be NULL if not + // supported by the interop. interop_callbacks_ = callbacks; return Result::kSuccess; } + void OnStreamChanged( + rtc::scoped_refptr stream) noexcept; + // // PeerConnectionObserver interface // @@ -359,12 +369,28 @@ class PeerConnectionImpl : public PeerConnection, const std::vector>& streams) noexcept override; + void OnTrack(rtc::scoped_refptr + transceiver) noexcept override; + /// Callback on remote track removed. void OnRemoveTrack(rtc::scoped_refptr receiver) noexcept override; void OnLocalDescCreated(webrtc::SessionDescriptionInterface* desc) noexcept; + // + // Internal + // + + void OnLocalTrackAddedToAudioTransceiver(AudioTransceiver& transceiver, + LocalAudioTrack& track) override; + void OnLocalTrackRemovedFromAudioTransceiver(AudioTransceiver& transceiver, + LocalAudioTrack& track) override; + void OnLocalTrackAddedToVideoTransceiver(VideoTransceiver& transceiver, + LocalVideoTrack& track) override; + void OnLocalTrackRemovedFromVideoTransceiver(VideoTransceiver& transceiver, + LocalVideoTrack& track) override; + /// The underlying PC object from the core implementation. This is NULL /// after |Close()| is called. rtc::scoped_refptr peer_; @@ -417,13 +443,21 @@ class PeerConnectionImpl : public PeerConnection, RenegotiationNeededCallback renegotiation_needed_callback_ RTC_GUARDED_BY(renegotiation_needed_callback_mutex_); - /// User callback invoked when a remote audio or video track is added. - TrackAddedCallback track_added_callback_ - RTC_GUARDED_BY(track_added_callback_mutex_); + /// User callback invoked when a remote audio track is added. + AudioTrackAddedCallback audio_track_added_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); - /// User callback invoked when a remote audio or video track is removed. - TrackRemovedCallback track_removed_callback_ - RTC_GUARDED_BY(track_removed_callback_mutex_); + /// User callback invoked when a remote audio track is removed. + AudioTrackRemovedCallback audio_track_removed_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); + + /// User callback invoked when a remote video track is added. + VideoTrackAddedCallback video_track_added_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); + + /// User callback invoked when a remote video track is removed. + VideoTrackRemovedCallback video_track_removed_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); std::mutex data_channel_added_callback_mutex_; std::mutex data_channel_removed_callback_mutex_; @@ -433,18 +467,34 @@ class PeerConnectionImpl : public PeerConnection, std::mutex ice_state_changed_callback_mutex_; std::mutex ice_gathering_state_changed_callback_mutex_; std::mutex renegotiation_needed_callback_mutex_; - std::mutex track_added_callback_mutex_; - std::mutex track_removed_callback_mutex_; + std::mutex media_track_callback_mutex_; + + std::unordered_map, + rtc::scoped_refptr> + remote_streams_; + + /// Collection of all transceivers of this peer connection. + std::vector> transceivers_ RTC_GUARDED_BY(tracks_mutex_); - rtc::scoped_refptr local_audio_track_; - rtc::scoped_refptr local_audio_sender_; - std::vector> remote_streams_; + /// Collection of all local audio tracks associated with this peer connection. + std::vector> local_audio_tracks_ + RTC_GUARDED_BY(tracks_mutex_); /// Collection of all local video tracks associated with this peer connection. std::vector> local_video_tracks_ RTC_GUARDED_BY(tracks_mutex_); - /// Mutex for all collections of all tracks. + /// Collection of all remote audio tracks associated with this peer + /// connection. + std::vector> remote_audio_tracks_ + RTC_GUARDED_BY(tracks_mutex_); + + /// Collection of all remote video tracks associated with this peer + /// connection. + std::vector> remote_video_tracks_ + RTC_GUARDED_BY(tracks_mutex_); + + /// Mutex for all collections of all tracks and transceivers. rtc::CriticalSection tracks_mutex_; /// Collection of all data channels associated with this peer connection. @@ -465,11 +515,6 @@ class PeerConnectionImpl : public PeerConnection, /// Mutex for data structures related to data channels. std::mutex data_channel_mutex_; - //< TODO - Clarify lifetime of those, for now same as this PeerConnection - std::unique_ptr local_audio_observer_; - std::unique_ptr remote_audio_observer_; - std::unique_ptr remote_video_observer_; - /// Flag to indicate if SCTP was negotiated during the initial SDP handshake /// (m=application), which allows subsequently to use data channels. If this /// is false then data channels will never connnect. This is set to true if a @@ -481,8 +526,88 @@ class PeerConnectionImpl : public PeerConnection, private: PeerConnectionImpl(const PeerConnectionImpl&) = delete; PeerConnectionImpl& operator=(const PeerConnectionImpl&) = delete; + + /// Insert a new transceiver wrapper at the given media line index. + Error InsertTransceiverAtMlineIndex(int mline_index, + RefPtr transceiver); + + /// Get an existing or create a new |AudioTransceiver| wrapper for a given RTP + /// sender of a local audio track. The sender should have an RTP transceiver + /// already, and this only takes care of finding/creating a wrapper for it, so + /// should never fail as long as the sender is indeed associated with this + /// peer connection. + ErrorOr> GetOrCreateAudioTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsAudioTransceiverInteropHandle transceiver_interop_handle); + + /// Get an existing or create a new |VideoTransceiver| wrapper for a given RTP + /// sender of a local video track. The sender should have an RTP transceiver + /// already, and this only takes care of finding/creating a wrapper for it, so + /// should never fail as long as the sender is indeed associated with this + /// peer connection. + ErrorOr> GetOrCreateVideoTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsVideoTransceiverInteropHandle transceiver_interop_handle); + + /// Get an existing or create a new |AudioTransceiver| wrapper for a given RTP + /// receiver of a newly added remote audio track. The receiver should have an + /// RTP transceiver already, and this only takes care of finding/creating a + /// wrapper for it, so should never fail as long as the receiver is indeed + /// associated with this peer connection. This is only called for new remote + /// tracks, so will only create new transceiver wrappers for some of the new + /// RTP transceivers. The callback on remote description applied will use + /// |GetOrCreateTransceiver()| to create the other ones. + ErrorOr> + GetOrCreateAudioTransceiverForNewRemoteTrack( + webrtc::RtpReceiverInterface* receiver); + + /// Get an existing or create a new |VideoTransceiver| wrapper for a given RTP + /// receiver of a newly added remote video track. The sender should have an + /// RTP transceiver already, and this only takes care of finding/creating a + /// wrapper for it, so should never fail as long as the receiver is indeed + /// associated with this peer connection. This is only called for new remote + /// tracks, so will only create new transceiver wrappers for some of the new + /// RTP transceivers. The callback on remote description applied will use + /// |GetOrCreateTransceiver()| to create the other ones. + ErrorOr> + GetOrCreateVideoTransceiverForRemoteNewTrack( + webrtc::RtpReceiverInterface* receiver); + + /// Get an existing or create a new |Transceiver| instance (either audio or + /// video) wrapper for a given RTP transceiver just created as part of a local + /// or remote description applied. This is always called for new transceivers, + /// even if no remote track is added, and will create the transceiver wrappers + /// not already created by the new remote track callbacks. + ErrorOr> GetOrCreateTransceiver( + int mline_index, + webrtc::RtpTransceiverInterface* rtp_transceiver, + std::string name); + + /// Create a new audio transceiver wrapper for an exist RTP transceiver + /// missing it. + ErrorOr> CreateAudioTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver); + + /// Create a new video transceiver wrapper for an exist RTP transceiver + /// missing it. + ErrorOr> CreateVideoTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver); + + static std::string ExtractTransceiverNameFromSender( + webrtc::RtpSenderInterface* sender); + + static std::string ExtractTransceiverNameFromReceiver( + webrtc::RtpReceiverInterface* receiver); }; +void StreamObserver::OnChanged() { + owner_.OnStreamChanged(stream_); +} + class CreateSessionDescObserver : public webrtc::CreateSessionDescriptionObserver { public: @@ -522,8 +647,9 @@ class SessionDescObserver : public webrtc::SetSessionDescriptionObserver { SessionDescObserver(Closure&& callback) : callback_(std::forward(callback)) {} void OnSuccess() override { - if (callback_) + if (callback_) { callback_(); + } } void OnFailure(webrtc::RTCError error) override { RTC_LOG(LS_ERROR) << "Error setting session description: " @@ -598,28 +724,104 @@ IceGatheringState IceGatheringStateFromImpl( return (IceGatheringState)impl_state; } +ErrorOr> PeerConnectionImpl::AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept { + if (IsClosed()) { + return Error(Result::kInvalidOperation, "The peer connection is closed."); + } + + std::string name; + if (!IsStringNullOrEmpty(config.name)) { + name = config.name; + } else { + name = rtc::CreateRandomUuid(); + } + if (!SdpIsValidToken(name)) { + rtc::StringBuilder str("Invalid video transceiver name: "); + str << name; + return Error(Result::kInvalidParameter, str.Release()); + } + + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + // Plan B doesn't have transceivers; just create a wrapper. + mline_index = (int)transceivers_.size(); // append + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), + config.transceiver_interop_handle); + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the low-level implementation object + webrtc::RtpTransceiverInit init{}; + init.direction = Transceiver::ToRtp(config.desired_direction); + init.stream_ids = Transceiver::DecodeStreamIDs(config.stream_ids); + if (!name.empty()) { + // Prepend transceiver name as first stream ID for track pairing + init.stream_ids.insert(init.stream_ids.begin(), name); + } + webrtc::RTCErrorOr> + ret = + peer_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, init); + if (!ret.ok()) { + return ErrorFromRTCError(ret.MoveError()); + } + rtc::scoped_refptr impl(ret.MoveValue()); + + // Find the mline index from the position inside the transceiver list + { + auto transceivers = peer_->GetTransceivers(); + auto it_tr = + std::find_if(transceivers.begin(), transceivers.end(), + [&impl](auto const& tr) { return (tr == impl); }); + RTC_DCHECK(it_tr != transceivers.end()); + mline_index = (int)std::distance(transceivers.begin(), it_tr); + } + + // Create the transceiver wrapper + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + config.transceiver_interop_handle); + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic."); + } + RTC_DCHECK(wrapper); + InsertTransceiverAtMlineIndex(mline_index, wrapper); + return wrapper; +} + ErrorOr> PeerConnectionImpl::AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept { + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept { if (IsClosed()) { - return Microsoft::MixedReality::WebRTC::Error( - Result::kInvalidOperation, "The peer connection is closed."); + return Error(Result::kInvalidOperation, "The peer connection is closed."); } auto result = peer_->AddTrack(video_track, {kAudioVideoStreamId}); - if (result.ok()) { - RefPtr track = - new LocalVideoTrack(global_factory_, *this, std::move(video_track), - std::move(result.MoveValue()), interop_handle); - { - rtc::CritScope lock(&tracks_mutex_); - local_video_tracks_.push_back(track); - } - return track; + if (!result.ok()) { + return ErrorFromRTCError(result.MoveError()); + } + rtc::scoped_refptr sender = result.MoveValue(); + ErrorOr> ret = + GetOrCreateVideoTransceiverForSender(sender, transceiver_interop_handle); + if (!ret.ok()) { + peer_->RemoveTrack(sender); + return ret.MoveError(); + } + RefPtr transceiver = ret.MoveValue(); + RefPtr track = new LocalVideoTrack( + global_factory_, *this, std::move(transceiver), std::move(video_track), + std::move(sender), track_interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + local_video_tracks_.push_back(track); } - return ErrorFromRTCError(result.MoveError()); + return track; } -webrtc::RTCError PeerConnectionImpl::RemoveLocalVideoTrack( +Result PeerConnectionImpl::RemoveLocalVideoTrack( LocalVideoTrack& video_track) noexcept { rtc::CritScope lock(&tracks_mutex_); auto it = std::find_if(local_video_tracks_.begin(), local_video_tracks_.end(), @@ -627,15 +829,13 @@ webrtc::RTCError PeerConnectionImpl::RemoveLocalVideoTrack( return track.get() == &video_track; }); if (it == local_video_tracks_.end()) { - return webrtc::RTCError( - webrtc::RTCErrorType::INVALID_PARAMETER, - "The video track is not associated with the peer connection."); + return Result::kInvalidParameter; } if (peer_) { video_track.RemoveFromPeerConnection(*peer_); } local_video_tracks_.erase(it); - return webrtc::RTCError::OK(); + return Result::kSuccess; } void PeerConnectionImpl::RemoveLocalVideoTracksFromSource( @@ -665,60 +865,119 @@ void PeerConnectionImpl::RemoveLocalVideoTracksFromSource( } } -void PeerConnectionImpl::SetLocalAudioTrackEnabled(bool enabled) noexcept { - if (local_audio_track_) { - local_audio_track_->set_enabled(enabled); +ErrorOr> PeerConnectionImpl::AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept { + if (IsClosed()) { + return Error(Result::kInvalidOperation, "The peer connection is closed."); } -} -bool PeerConnectionImpl::IsLocalAudioTrackEnabled() const noexcept { - if (local_audio_track_) { - return local_audio_track_->enabled(); + std::string name; + if (!IsStringNullOrEmpty(config.name)) { + name = config.name; + } else { + name = rtc::CreateRandomUuid(); + } + if (!SdpIsValidToken(name)) { + rtc::StringBuilder str("Invalid audio transceiver name: "); + str << name; + return Error(Result::kInvalidParameter, str.Release()); + } + + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + // Plan B doesn't have transceivers; just create a wrapper. + mline_index = (int)transceivers_.size(); // append + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), + config.transceiver_interop_handle); + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the low-level implementation object + webrtc::RtpTransceiverInit init{}; + init.direction = Transceiver::ToRtp(config.desired_direction); + init.stream_ids = Transceiver::DecodeStreamIDs(config.stream_ids); + if (!name.empty()) { + // Prepend transceiver name as first stream ID for track pairing + init.stream_ids.insert(init.stream_ids.begin(), name); + } + webrtc::RTCErrorOr> + ret = + peer_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_AUDIO, init); + if (!ret.ok()) { + return ErrorFromRTCError(ret.MoveError()); + } + rtc::scoped_refptr impl(ret.MoveValue()); + + // Find the mline index from the position inside the transceiver list + { + auto transceivers = peer_->GetTransceivers(); + auto it_tr = + std::find_if(transceivers.begin(), transceivers.end(), + [&impl](auto const& tr) { return (tr == impl); }); + RTC_DCHECK(it_tr != transceivers.end()); + mline_index = (int)std::distance(transceivers.begin(), it_tr); + } + + // Create the transceiver wrapper + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + config.transceiver_interop_handle); + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic."); } - return false; + RTC_DCHECK(wrapper); + InsertTransceiverAtMlineIndex(mline_index, wrapper); + return wrapper; } -bool PeerConnectionImpl::AddLocalAudioTrack( - rtc::scoped_refptr audio_track) noexcept { - if (local_audio_track_) { - return false; +ErrorOr> PeerConnectionImpl::AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept { + if (IsClosed()) { + return Microsoft::MixedReality::WebRTC::Error( + Result::kInvalidOperation, "The peer connection is closed."); } - if (local_audio_sender_) { - // Reuse the existing sender. - if (local_audio_sender_->SetTrack(audio_track.get())) { - if (auto* sink = local_audio_observer_.get()) { - // FIXME - Current implementation of AddSink() for the local audio - // capture device is no-op. So this callback is never fired. - audio_track->AddSink(sink); - } - local_audio_track_ = std::move(audio_track); - return true; - } - } else if (peer_) { - // Create a new sender. - auto result = peer_->AddTrack(audio_track, {kAudioVideoStreamId}); - if (result.ok()) { - if (auto* sink = local_audio_observer_.get()) { - // FIXME - Current implementation of AddSink() for the local audio - // capture device is no-op. So this callback is never fired. - audio_track->AddSink(sink); - } - local_audio_sender_ = result.value(); - local_audio_track_ = std::move(audio_track); - return true; - } + auto result = peer_->AddTrack(audio_track, {kAudioVideoStreamId}); + if (!result.ok()) { + return ErrorFromRTCError(result.MoveError()); + } + rtc::scoped_refptr sender = result.MoveValue(); + ErrorOr> ret = + GetOrCreateAudioTransceiverForSender(sender, transceiver_interop_handle); + if (!ret.ok()) { + peer_->RemoveTrack(sender); + return ret.MoveError(); + } + RefPtr transceiver = ret.MoveValue(); + RefPtr track = new LocalAudioTrack( + global_factory_, *this, std::move(transceiver), std::move(audio_track), + std::move(sender), track_interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + local_audio_tracks_.push_back(track); } - return false; + return track; } -void PeerConnectionImpl::RemoveLocalAudioTrack() noexcept { - if (!local_audio_track_) - return; - if (auto* sink = local_audio_observer_.get()) { - local_audio_track_->RemoveSink(sink); +Result PeerConnectionImpl::RemoveLocalAudioTrack( + LocalAudioTrack& audio_track) noexcept { + rtc::CritScope lock(&tracks_mutex_); + auto it = std::find_if(local_audio_tracks_.begin(), local_audio_tracks_.end(), + [&audio_track](const RefPtr& track) { + return track.get() == &audio_track; + }); + if (it == local_audio_tracks_.end()) { + return Result::kInvalidParameter; + } + if (peer_) { + audio_track.RemoveFromPeerConnection(*peer_); } - local_audio_sender_->SetTrack(nullptr); - local_audio_track_ = nullptr; + local_audio_tracks_.erase(it); + return Result::kSuccess; } ErrorOr> PeerConnectionImpl::AddDataChannel( @@ -886,6 +1145,19 @@ void PeerConnectionImpl::OnDataChannelAdded( } } +void PeerConnectionImpl::OnStreamChanged( + rtc::scoped_refptr stream) noexcept { + webrtc::AudioTrackVector audio_tracks = stream->GetAudioTracks(); + webrtc::VideoTrackVector video_tracks = stream->GetVideoTracks(); + RTC_LOG(LS_INFO) << "Media stream #" << stream->id() + << " changed: " << audio_tracks.size() + << " audio tracks and " << video_tracks.size() + << " video tracks."; + // for (auto&& audio_track : remote_audio_tracks_) { + // if (audio_track == audio_tracks[0]) + //} +} + bool PeerConnectionImpl::AddIceCandidate(const char* sdp_mid, const int sdp_mline_index, const char* candidate) noexcept { @@ -908,18 +1180,13 @@ bool PeerConnectionImpl::CreateOffer() noexcept { if (!peer_) { return false; } - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; - /*if (mandatory_receive_)*/ { //< TODO - This is legacy, should use - // transceivers - options.offer_to_receive_audio = true; - options.offer_to_receive_video = true; - } { auto lock = std::scoped_lock{data_channel_mutex_}; if (data_channels_.empty()) { sctp_negotiated_ = false; } } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options{}; auto observer = new rtc::RefCountedObject(this); // 0 ref peer_->CreateOffer(observer, options); @@ -931,12 +1198,7 @@ bool PeerConnectionImpl::CreateAnswer() noexcept { if (!peer_) { return false; } - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; - /*if (mandatory_receive_)*/ { //< TODO - This is legacy, should use - // transceivers - options.offer_to_receive_audio = true; - options.offer_to_receive_video = true; - } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options{}; auto observer = new rtc::RefCountedObject(this); // 0 ref peer_->CreateAnswer(observer, options); @@ -952,33 +1214,57 @@ void PeerConnectionImpl::Close() noexcept { // Close the connection peer_->Close(); - // Remove local tracks { rtc::CritScope lock(&tracks_mutex_); + + // Remove local tracks while (!local_video_tracks_.empty()) { RefPtr& ptr = local_video_tracks_.back(); RemoveLocalVideoTrack(*ptr); } - } + while (!local_audio_tracks_.empty()) { + RefPtr& ptr = local_audio_tracks_.back(); + RemoveLocalAudioTrack(*ptr); + } - // Ensure that observers (sinks) are removed, otherwise the media pipelines - // will continue to try to feed them with data after they're destroyed - // RemoveLocalVideoTrack(); TODO - do we need to keep a list of local tracks - // and do something here? - RemoveLocalAudioTrack(); - local_audio_sender_ = nullptr; - for (auto stream : remote_streams_) { - if (auto* sink = remote_video_observer_.get()) { - for (auto&& video_track : stream->GetVideoTracks()) { - video_track->RemoveSink(sink); + // Force-remove remote tracks. It doesn't look like the TrackRemoved + // callback is called when Close() is used, so force it here. + auto rat = std::move(remote_audio_tracks_); + auto rvt = std::move(remote_video_tracks_); + auto cb_lock = std::scoped_lock{media_track_callback_mutex_}; + auto audio_cb = audio_track_removed_callback_; + for (auto&& track : rat) { + track->OnTrackRemoved(*this); + if (auto interop_handle = track->GetInteropHandle()) { + if (audio_cb) { + auto transceiver = track->GetTransceiver(); + auto transceiver_interop_handle = transceiver->GetInteropHandle(); + audio_cb(interop_handle, track.get(), transceiver_interop_handle, + transceiver); + } } } - if (auto* sink = remote_audio_observer_.get()) { - for (auto&& audio_track : stream->GetAudioTracks()) { - audio_track->RemoveSink(sink); + auto video_cb = video_track_removed_callback_; + for (auto&& track : rvt) { + track->OnTrackRemoved(*this); + if (auto interop_handle = track->GetInteropHandle()) { + if (video_cb) { + auto transceiver = track->GetTransceiver(); + auto transceiver_interop_handle = transceiver->GetInteropHandle(); + video_cb(interop_handle, track.get(), transceiver_interop_handle, + transceiver); + } } } + + // Clear transceivers + //< TODO - This is done inside the lock, but the lock is released before + // peer_ is cleared, so before the connection is actually closed, which + // doesn't prevent add(Audio|Video)Transceiver from being called again in + // parallel... + transceivers_.clear(); } + remote_streams_.clear(); RemoveAllDataChannels(); @@ -1020,6 +1306,22 @@ bool PeerConnectionImpl::SetRemoteDescriptionAsync( return false; rtc::scoped_refptr observer = new rtc::RefCountedObject([this, callback] { + // Inspect transceiver directions, check for changes to update the + // interop layer with the actually negotiated direction. + std::vector> + changed_transceivers; + int mline_index = 0; // native transceivers are in mline_index order + for (auto&& tr : peer_->GetTransceivers()) { + // If transceiver is created from the result of applying a remote + // description, then the transceiver name is extracted from the + // receiver, in an attempt to pair with the remote peer's track. + std::string name = ExtractTransceiverNameFromReceiver(tr->receiver()); + ErrorOr> err = + GetOrCreateTransceiver(mline_index, tr, std::move(name)); + RTC_DCHECK(err.ok()); + err.value()->OnSessionDescUpdated(/*remote=*/true); + ++mline_index; + } // Fire completed callback to signal remote description was applied. callback(); }); @@ -1058,7 +1360,9 @@ void PeerConnectionImpl::OnAddStream( RTC_LOG(LS_INFO) << "Added stream #" << stream->id() << " with " << stream->GetAudioTracks().size() << " audio tracks and " << stream->GetVideoTracks().size() << " video tracks."; - remote_streams_.push_back(stream); + auto observer = std::make_unique(*this, stream); + stream->RegisterObserver(observer.get()); + remote_streams_.emplace(std::move(observer), stream); } void PeerConnectionImpl::OnRemoveStream( @@ -1066,10 +1370,17 @@ void PeerConnectionImpl::OnRemoveStream( RTC_LOG(LS_INFO) << "Removed stream #" << stream->id() << " with " << stream->GetAudioTracks().size() << " audio tracks and " << stream->GetVideoTracks().size() << " video tracks."; - auto it = std::find(remote_streams_.begin(), remote_streams_.end(), stream); + auto it = std::find_if( + remote_streams_.begin(), remote_streams_.end(), + [&stream]( + std::pair, + rtc::scoped_refptr>& pair) { + return pair.second == stream; + }); if (it == remote_streams_.end()) { return; } + stream->UnregisterObserver(it->first.get()); remote_streams_.erase(it); } @@ -1188,47 +1499,145 @@ void PeerConnectionImpl::OnAddTrack( rtc::scoped_refptr receiver, const std::vector>& /*streams*/) noexcept { - RTC_LOG(LS_INFO) << "Added track #" << receiver->id() << " of type " + RTC_LOG(LS_INFO) << "Added receiver #" << receiver->id() << " of type " << (int)receiver->media_type(); for (auto&& stream : receiver->streams()) { - RTC_LOG(LS_INFO) << "+ Track #" << receiver->id() << " with stream #" - << stream->id(); + RTC_LOG(LS_INFO) << "+ Track #" << receiver->track()->id() + << " with stream #" << stream->id(); } - // Register the remote observer + // Create the remote track wrapper rtc::scoped_refptr track = receiver->track(); - TrackKind trackKind = TrackKind::kUnknownTrack; - const std::string& trackKindStr = track->kind(); - if (trackKindStr == webrtc::MediaStreamTrackInterface::kAudioKind) { - trackKind = TrackKind::kAudioTrack; - if (auto* sink = remote_audio_observer_.get()) { - auto audio_track = static_cast(track.get()); - audio_track->AddSink(sink); + const std::string& track_name = track->id(); + const std::string& track_kind_str = track->kind(); + if (track_kind_str == webrtc::MediaStreamTrackInterface::kAudioKind) { + rtc::scoped_refptr audio_track( + static_cast(track.release())); + + // Create an interop wrapper for the new native object if needed + mrsRemoteAudioTrackInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.remote_audio_track_create_object) { + mrsRemoteAudioTrackConfig config; + config.track_name = track_name.c_str(); + interop_handle = (*create_cb)(interop_handle_, config); } - } else if (trackKindStr == webrtc::MediaStreamTrackInterface::kVideoKind) { - trackKind = TrackKind::kVideoTrack; - if (auto* sink = remote_video_observer_.get()) { - rtc::VideoSinkWants sink_settings{}; - sink_settings.rotation_applied = - true; // no exposed API for caller to handle rotation - auto video_track = static_cast(track.get()); - video_track->AddOrUpdateSink(sink, sink_settings); + + // Get or create the transceiver wrapper based on the RTP receiver. Because + // this callback is fired before the one at the end of the remote + // description being applied, the transceiver wrappers for the newly added + // RTP transceivers have not been created yet, so create them here. + auto ret = GetOrCreateAudioTransceiverForNewRemoteTrack(receiver); + if (!ret.ok()) { + return; } - } else { - return; - } + RefPtr transceiver = ret.MoveValue(); - // Invoke the TrackAdded callback - { - auto lock = std::scoped_lock{track_added_callback_mutex_}; - auto cb = track_added_callback_; - if (cb) { - cb(trackKind); + // The transceiver wrapper might have been created, in which case we need to + // inform its interop wrapper of its handle. + mrsAudioTransceiverInteropHandle transceiver_interop_handle = + transceiver->GetInteropHandle(); + + // Create the native object + RefPtr remote_audio_track = new RemoteAudioTrack( + global_factory_, *this, transceiver, std::move(audio_track), + std::move(receiver), interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + remote_audio_tracks_.emplace_back(remote_audio_track); + } + + // Invoke the AudioTrackAdded callback, which will set the native handle on + // the interop wrapper (if created above) + { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = audio_track_added_callback_; + if (cb) { + AudioTransceiverHandle tranceiver_handle = transceiver.release(); + RemoteAudioTrackHandle audio_handle = remote_audio_track.release(); + cb(interop_handle, audio_handle, transceiver_interop_handle, + tranceiver_handle); + } + } + } else if (track_kind_str == webrtc::MediaStreamTrackInterface::kVideoKind) { + rtc::scoped_refptr video_track( + static_cast(track.release())); + + // Create an interop wrapper for the new native object if needed + mrsRemoteVideoTrackInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.remote_video_track_create_object) { + mrsRemoteVideoTrackConfig config; + config.track_name = track_name.c_str(); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Get or create the transceiver wrapper based on the RTP receiver. Because + // this callback is fired before the one at the end of the remote + // description being applied, the transceiver wrappers for the newly added + // RTP transceivers have not been created yet, so create them here. + auto ret = GetOrCreateVideoTransceiverForRemoteNewTrack(receiver); + if (!ret.ok()) { + return; + } + RefPtr transceiver = ret.MoveValue(); + + // The transceiver wrapper might have been created, in which case we need to + // inform its interop wrapper of its handle. + mrsVideoTransceiverInteropHandle transceiver_interop_handle = + transceiver->GetInteropHandle(); + + // Create the native object + RefPtr remote_video_track = new RemoteVideoTrack( + global_factory_, *this, transceiver, std::move(video_track), + std::move(receiver), interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + remote_video_tracks_.emplace_back(remote_video_track); + } + + // Invoke the VideoTrackAdded callback, which will set the native handle on + // the interop wrapper (if created above) + { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = video_track_added_callback_; + if (cb) { + VideoTransceiverHandle tranceiver_handle = transceiver.release(); + RemoteVideoTrackHandle video_handle = remote_video_track.release(); + cb(interop_handle, video_handle, transceiver_interop_handle, + tranceiver_handle); + } } } } +void PeerConnectionImpl::OnTrack( + rtc::scoped_refptr transceiver) noexcept { + RTC_LOG(LS_INFO) << "Added transceiver mid=#" << transceiver->mid().value() + << " of type " << (int)transceiver->media_type() + << " with desired direction " + << (int)transceiver->direction(); + auto receiver = transceiver->receiver(); + if (auto track = receiver->track()) { + RTC_LOG(LS_INFO) << "Recv with track #" << track->id() + << " enabled=" << track->enabled(); + } else { + RTC_LOG(LS_INFO) << "Recv with NULL track"; + } + for (auto&& id : receiver->stream_ids()) { + RTC_LOG(LS_INFO) << "+ Stream #" << id; + } + auto sender = transceiver->sender(); + if (auto track = sender->track()) { + RTC_LOG(LS_INFO) << "Send #" << track->id() + << " enabled=" << track->enabled(); + } else { + RTC_LOG(LS_INFO) << "Send with NULL track"; + } + for (auto&& id : sender->stream_ids()) { + RTC_LOG(LS_INFO) << "+ Stream #" << id; + } +} + void PeerConnectionImpl::OnRemoveTrack( rtc::scoped_refptr receiver) noexcept { RTC_LOG(LS_INFO) << "Removed track #" << receiver->id() << " of type " @@ -1241,31 +1650,59 @@ void PeerConnectionImpl::OnRemoveTrack( // Unregister the remote observer rtc::scoped_refptr track = receiver->track(); - TrackKind trackKind = TrackKind::kUnknownTrack; - const std::string& trackKindStr = track->kind(); - if (trackKindStr == webrtc::MediaStreamTrackInterface::kAudioKind) { - trackKind = TrackKind::kAudioTrack; - if (auto* sink = remote_audio_observer_.get()) { - auto audio_track = static_cast(track.get()); - audio_track->RemoveSink(sink); + const std::string& track_kind_str = track->kind(); + if (track_kind_str == webrtc::MediaStreamTrackInterface::kAudioKind) { + rtc::CritScope tracks_lock(&tracks_mutex_); + auto it = + std::find_if(remote_audio_tracks_.begin(), remote_audio_tracks_.end(), + [&receiver](const RefPtr& remote_track) { + return (remote_track->receiver() == receiver); + }); + if (it == remote_audio_tracks_.end()) { + return; } - } else if (trackKindStr == webrtc::MediaStreamTrackInterface::kVideoKind) { - trackKind = TrackKind::kVideoTrack; - if (auto* sink = remote_video_observer_.get()) { - auto video_track = static_cast(track.get()); - video_track->RemoveSink(sink); + RefPtr audio_track = std::move(*it); + RefPtr audio_transceiver = audio_track->GetTransceiver(); + remote_audio_tracks_.erase(it); + audio_track->OnTrackRemoved(*this); + + // Invoke the TrackRemoved callback + if (auto interop_handle = audio_track->GetInteropHandle()) { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = audio_track_removed_callback_; + if (cb) { + auto transceiver_interop_handle = audio_transceiver->GetInteropHandle(); + cb(interop_handle, audio_track.get(), transceiver_interop_handle, + audio_transceiver.get()); + } } - } else { - return; - } - - // Invoke the TrackRemoved callback - { - auto lock = std::scoped_lock{track_removed_callback_mutex_}; - auto cb = track_removed_callback_; - if (cb) { - cb(trackKind); + // |audio_track| goes out of scope and destroys the C++ instance + } else if (track_kind_str == webrtc::MediaStreamTrackInterface::kVideoKind) { + rtc::CritScope tracks_lock(&tracks_mutex_); + auto it = + std::find_if(remote_video_tracks_.begin(), remote_video_tracks_.end(), + [&receiver](const RefPtr& remote_track) { + return (remote_track->receiver() == receiver); + }); + if (it == remote_video_tracks_.end()) { + return; + } + RefPtr video_track = std::move(*it); + RefPtr video_transceiver = video_track->GetTransceiver(); + remote_video_tracks_.erase(it); + video_track->OnTrackRemoved(*this); + + // Invoke the TrackRemoved callback + if (auto interop_handle = video_track->GetInteropHandle()) { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = video_track_removed_callback_; + if (cb) { + auto transceiver_interop_handle = video_transceiver->GetInteropHandle(); + cb(interop_handle, video_track.get(), transceiver_interop_handle, + video_transceiver.get()); + } } + // |video_track| goes out of scope and destroys the C++ instance } } @@ -1274,28 +1711,544 @@ void PeerConnectionImpl::OnLocalDescCreated( if (!peer_) { return; } - auto lock = std::scoped_lock{local_sdp_ready_to_send_callback_mutex_}; - auto cb = local_sdp_ready_to_send_callback_; - rtc::scoped_refptr observer; - if (cb) { - std::string type{SdpTypeToString(desc->GetType())}; - ensureNullTerminatedCString(type); - std::string sdp; - desc->ToString(&sdp); - ensureNullTerminatedCString(sdp); - observer = new rtc::RefCountedObject( - [cb, type = std::move(type), sdp = std::move(sdp)] { - cb(type.c_str(), sdp.c_str()); - }); - } else { - observer = new rtc::RefCountedObject(); - } + rtc::scoped_refptr observer = + new rtc::RefCountedObject([this] { + // Inspect transceiver directions, check for changes to update the + // interop layer with the actually negotiated direction. + std::vector> + changed_transceivers; + int mline_index = 0; // native transceivers are in mline_index order + for (auto&& tr : peer_->GetTransceivers()) { + // If transceiver is created from the result of applying a local + // description, then the transceiver name is extracted from the + // sender, as the name should have been set by the user. + std::string name = ExtractTransceiverNameFromSender(tr->sender()); + ErrorOr> err = + GetOrCreateTransceiver(mline_index, tr, std::move(name)); + RTC_DCHECK(err.ok()); + err.value()->OnSessionDescUpdated(/*remote=*/false); + ++mline_index; + } + + // Fire interop callback, if any + { + auto lock = std::scoped_lock{local_sdp_ready_to_send_callback_mutex_}; + if (auto cb = local_sdp_ready_to_send_callback_) { + auto desc = peer_->local_description(); + std::string type{SdpTypeToString(desc->GetType())}; + std::string sdp; + desc->ToString(&sdp); + cb(type.c_str(), sdp.c_str()); + } + } + }); // SetLocalDescription will invoke observer.OnSuccess() once done, which // will in turn invoke the |local_sdp_ready_to_send_callback_| registered if // any, or do nothing otherwise. The observer is a mandatory parameter. peer_->SetLocalDescription(observer, desc); } +void PeerConnectionImpl::OnLocalTrackAddedToAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kAudio)); + }) != transceivers_.end()); + RTC_DCHECK(std::find_if(local_audio_tracks_.begin(), + local_audio_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }) == local_audio_tracks_.end()); + local_audio_tracks_.push_back(&track); +} + +void PeerConnectionImpl::OnLocalTrackRemovedFromAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kAudio)); + }) != transceivers_.end()); + auto it = std::find_if(local_audio_tracks_.begin(), local_audio_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }); + RTC_DCHECK(it != local_audio_tracks_.end()); + local_audio_tracks_.erase(it); +} + +void PeerConnectionImpl::OnLocalTrackAddedToVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kVideo)); + }) != transceivers_.end()); + RTC_DCHECK(std::find_if(local_video_tracks_.begin(), + local_video_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }) == local_video_tracks_.end()); + local_video_tracks_.push_back(&track); +} + +void PeerConnectionImpl::OnLocalTrackRemovedFromVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kVideo)); + }) != transceivers_.end()); + auto it = std::find_if(local_video_tracks_.begin(), local_video_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }); + RTC_DCHECK(it != local_video_tracks_.end()); + local_video_tracks_.erase(it); +} + +Error PeerConnectionImpl::InsertTransceiverAtMlineIndex( + int mline_index, + RefPtr transceiver) { + RTC_CHECK(mline_index >= 0); + rtc::CritScope lock(&tracks_mutex_); + if (mline_index >= transceivers_.size()) { + // Insert empty entries for now; they should be filled when processing + // other added remote tracks or when finishing the transceiver status + // update. + while (mline_index >= transceivers_.size() + 1) { + transceivers_.push_back(nullptr); + } + transceivers_.push_back(transceiver); + } else { + if (transceivers_[mline_index]) { + RTC_LOG(LS_ERROR) << "Trying to insert transceiver (name=" + << transceiver->GetName().c_str() + << ") at mline index #" << mline_index + << ", but another transceiver (name=" + << transceivers_[mline_index]->GetName().c_str() + << ") already exists with the same index."; + return Error(Result::kUnknownError, + "Duplicate transceiver for mline index"); + } + transceivers_[mline_index] = transceiver; + } + return Error(Result::kSuccess); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateAudioTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsAudioTransceiverInteropHandle transceiver_interop_handle) { + RTC_DCHECK(sender->media_type() == cricket::MediaType::MEDIA_TYPE_AUDIO); + + // Find existing transceiver for sender + auto it = std::find_if(transceivers_.begin(), transceivers_.end(), + [sender](const RefPtr& tr) { + return (tr && (tr->impl()->sender() == sender)); + }); + if (it != transceivers_.end()) { + RTC_DCHECK(MediaKind::kAudio == it->get()->GetMediaKind()); + RefPtr audio_transceiver = + static_cast(it->get()); + RTC_DCHECK_EQ(transceiver_interop_handle, + audio_transceiver->GetInteropHandle()); + return audio_transceiver; + } + + std::string name = ExtractTransceiverNameFromSender(sender); + + // Create new transceiver wrapper for a newly created local audio track. + // This is called from AddLocalAudioTrack() only, after calling the lower + // level AddTrack() which may reuse an existing transceiver, so trust the + // implementation for the mline index. + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Find transceiver implementation. It doesn't seem like there is a direct + // back-link, so iterate over all the peer connection transceivers. + auto transceivers = peer_->GetTransceivers(); + auto it_tr = std::find_if( + transceivers.begin(), transceivers.end(), + [sender](auto const& tr) { return (tr->sender() == sender); }); + if (it_tr == transceivers.end()) { + return Error(Result::kInvalidOperation, + "Cannot match RTP sender with RTP transceiver."); + } + rtc::scoped_refptr impl = *it_tr; + mline_index = (int)std::distance(transceivers.begin(), it_tr); + + // Create the transceiver wrapper + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + transceiver_interop_handle); + + // Note: at this point the native wrapper knows about the interop wrapper, + // but not the opposite. Normally we'd fire another "created-callback" + // with the native wrapper handle to sync the interop wrapper, but here it + // is being created as part of a local track creation, so we bundle that + // with the "track-created" event. + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (wrapper) { + auto err = InsertTransceiverAtMlineIndex(mline_index, wrapper); + if (!err.ok()) { + return err; + } + return wrapper; + } + return Error(Result::kUnknownError, + "Failed to create a new transceiver for local audio track."); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateVideoTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsVideoTransceiverInteropHandle transceiver_interop_handle) { + RTC_DCHECK(sender->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO); + + // Find existing transceiver for sender + auto it = std::find_if(transceivers_.begin(), transceivers_.end(), + [sender](const RefPtr& tr) { + return (tr && (tr->impl()->sender() == sender)); + }); + if (it != transceivers_.end()) { + RTC_DCHECK(MediaKind::kVideo == it->get()->GetMediaKind()); + RefPtr video_transceiver = + static_cast(it->get()); + RTC_DCHECK_EQ(transceiver_interop_handle, + video_transceiver->GetInteropHandle()); + return video_transceiver; + } + + std::string name = ExtractTransceiverNameFromSender(sender); + + // Create new transceiver for video track + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Find transceiver implementation. It doesn't seem like there is a direct + // back-link, so iterate over all the peer connection transceivers. + auto transceivers = peer_->GetTransceivers(); + auto it_tr = std::find_if( + transceivers.begin(), transceivers.end(), + [sender](auto const& tr) { return (tr->sender() == sender); }); + if (it_tr == transceivers.end()) { + return Error(Result::kInvalidOperation, + "Cannot match RTP sender with RTP transceiver."); + } + rtc::scoped_refptr impl = *it_tr; + mline_index = (int)std::distance(transceivers.begin(), it_tr); + + // Create the transceiver wrapper + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + transceiver_interop_handle); + + // Note: at this point the native wrapper knows about the interop wrapper, + // but not the opposite. Normally we'd fire another "created-callback" + // with the native wrapper handle to sync the interop wrapper, but here it + // is being created as part of a local track creation, so we bundle that + // with the "track-created" event. + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (wrapper) { + auto err = InsertTransceiverAtMlineIndex(mline_index, wrapper); + if (!err.ok()) { + return err; + } + return wrapper; + } + return Error(Result::kUnknownError, + "Failed to create a new transceiver for local video track."); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateAudioTransceiverForNewRemoteTrack( + webrtc::RtpReceiverInterface* receiver) { + RTC_DCHECK(receiver->media_type() == cricket::MediaType::MEDIA_TYPE_AUDIO); + + // Try to find an existing audio transceiver wrapper for the given RTP + // receiver of the remote track. + { + auto it_tr = std::find_if(transceivers_.begin(), transceivers_.end(), + [receiver](const RefPtr& tr) { + return (tr->impl()->receiver() == receiver); + }); + if (it_tr != transceivers_.end()) { + RTC_DCHECK(MediaKind::kAudio == (*it_tr)->GetMediaKind()); + RefPtr audio_transceiver = + static_cast(it_tr->get()); + return audio_transceiver; + } + } + + // The new remote track should already have a low-level implementation RTP + // transceiver from applying the remote description. But the wrapper for it + // was not created yet. Find the RTP transceiver of the RTP receiver, bearing + // in mind its mline index is not necessarily contiguous in the wrapper array. + auto transceivers = peer_->GetTransceivers(); + auto it_impl = std::find_if( + transceivers.begin(), transceivers.end(), + [receiver](auto&& tr) { return tr->receiver() == receiver; }); + if (it_impl == transceivers.end()) { + return Error( + Result::kNotFound, + "Failed to match RTP receiver with an existing RTP transceiver."); + } + rtc::scoped_refptr impl = *it_impl; + const int mline_index = (int)std::distance(transceivers.begin(), it_impl); + + std::string name = ExtractTransceiverNameFromReceiver(receiver); + + // Create a new audio transceiver wrapper for it + return CreateAudioTransceiver(mline_index, std::move(name), std::move(impl)); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateVideoTransceiverForRemoteNewTrack( + webrtc::RtpReceiverInterface* receiver) { + RTC_DCHECK(receiver->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO); + + // Try to find an existing video transceiver wrapper for the given RTP + // receiver of the remote track. + { + auto it_tr = std::find_if(transceivers_.begin(), transceivers_.end(), + [receiver](const RefPtr& tr) { + return (tr->impl()->receiver() == receiver); + }); + if (it_tr != transceivers_.end()) { + RTC_DCHECK(MediaKind::kVideo == (*it_tr)->GetMediaKind()); + RefPtr video_transceiver = + static_cast(it_tr->get()); + return video_transceiver; + } + } + + // The new remote track should already have a low-level implementation RTP + // transceiver from applying the remote description. But the wrapper for it + // was not created yet. Find the RTP transceiver of the RTP receiver, bearing + // in mind its mline index is not necessarily contiguous in the wrapper array. + auto transceivers = peer_->GetTransceivers(); + auto it_impl = std::find_if( + transceivers.begin(), transceivers.end(), + [receiver](auto&& tr) { return tr->receiver() == receiver; }); + if (it_impl == transceivers.end()) { + return Error( + Result::kNotFound, + "Failed to match RTP receiver with an existing RTP transceiver."); + } + rtc::scoped_refptr impl = *it_impl; + const int mline_index = (int)std::distance(transceivers.begin(), it_impl); + + std::string name = ExtractTransceiverNameFromReceiver(receiver); + + // Create a new video transceiver wrapper for it + return CreateVideoTransceiver(mline_index, std::move(name), std::move(impl)); +} + +ErrorOr> PeerConnectionImpl::GetOrCreateTransceiver( + int mline_index, + webrtc::RtpTransceiverInterface* rtp_transceiver, + std::string name) { + switch (rtp_transceiver->media_type()) { + case cricket::MediaType::MEDIA_TYPE_AUDIO: { + // Find an existing transceiver wrapper which would have been created just + // a moment ago by the remote track added callback. + rtc::CritScope lock(&tracks_mutex_); + for (auto&& tr : transceivers_) { + if (tr && (tr->impl() == rtp_transceiver)) { + RTC_DCHECK(MediaKind::kAudio == tr->GetMediaKind()); + return static_cast>(tr); + } + } + // Not found - create a new one + return CreateAudioTransceiver(mline_index, std::move(name), + rtp_transceiver); + }; + + case cricket::MediaType::MEDIA_TYPE_VIDEO: { + // Find an existing transceiver wrapper which would have been created just + // a moment ago by the remote track added callback. + rtc::CritScope lock(&tracks_mutex_); + for (auto&& tr : transceivers_) { + if (tr && (tr->impl() == rtp_transceiver)) { + RTC_DCHECK(MediaKind::kVideo == tr->GetMediaKind()); + return static_cast>(tr); + } + } + // Not found - create a new one + return CreateVideoTransceiver(mline_index, std::move(name), + rtp_transceiver); + }; + + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } +} + +ErrorOr> PeerConnectionImpl::CreateAudioTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver) { + // Create an interop wrapper for the new native object if needed + mrsAudioTransceiverInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.audio_transceiver_create_object) { + mrsAudioTransceiverConfig config{}; + config.name = name.c_str(); + config.mline_index = mline_index; + config.initial_desired_direction = + Transceiver::FromRtp(rtp_transceiver->direction()); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Create new transceiver wrapper + RefPtr transceiver; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the transceiver wrapper + transceiver = new AudioTransceiver( + global_factory_, *this, mline_index, std::move(name), + std::move(rtp_transceiver), interop_handle); + + // Synchronize the interop wrapper with the current object. + if (auto cb = interop_callbacks_.audio_transceiver_finish_create) { + transceiver->AddRef(); + cb(interop_handle, transceiver.get()); + } + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (transceiver) { + auto err = InsertTransceiverAtMlineIndex(mline_index, transceiver); + if (!err.ok()) { + return err; + } + } + return transceiver; +} + +ErrorOr> PeerConnectionImpl::CreateVideoTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver) { + // Create an interop wrapper for the new native object if needed + mrsVideoTransceiverInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.video_transceiver_create_object) { + mrsVideoTransceiverConfig config{}; + config.name = name.c_str(); + config.mline_index = mline_index; + config.initial_desired_direction = + Transceiver::FromRtp(rtp_transceiver->direction()); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Create new transceiver wrapper + RefPtr transceiver; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the transceiver wrapper + transceiver = new VideoTransceiver( + global_factory_, *this, mline_index, std::move(name), + std::move(rtp_transceiver), interop_handle); + + // Synchronize the interop wrapper with the current object. + if (auto cb = interop_callbacks_.video_transceiver_finish_create) { + transceiver->AddRef(); + cb(interop_handle, transceiver.get()); + } + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (transceiver) { + auto err = InsertTransceiverAtMlineIndex(mline_index, transceiver); + if (!err.ok()) { + return err; + } + } + return transceiver; +} + +std::string PeerConnectionImpl::ExtractTransceiverNameFromSender( + webrtc::RtpSenderInterface* sender) { + // Find the pairing name as the first stream ID. + // See |LocalAudioTrack::GetName()|, |RemoteAudioTrack::GetName()|, + // |LocalVideoTrack::GetName()|, |RemoteVideoTrack::GetName()|. + auto ids = sender->stream_ids(); + if (!ids.empty()) { + return ids[0]; + } + // Fallback on track's ID, even though it's not pairable in Unified Plan (and + // technically neither in Plan B, although this works in practice). + if (rtc::scoped_refptr track = + sender->track()) { + return track->id(); + } + return {}; +} + +std::string PeerConnectionImpl::ExtractTransceiverNameFromReceiver( + webrtc::RtpReceiverInterface* receiver) { + // Find the pairing name as the first stream ID. + // See |LocalAudioTrack::GetName()|, |RemoteAudioTrack::GetName()|, + // |LocalVideoTrack::GetName()|, |RemoteVideoTrack::GetName()|. + { + // BUG + // webrtc::RtpReceiverInterface::stream_ids() is not proxied correctly, does + // not resolve to its implementation and instead always returns an empty + // vector. Use ::streams() instead even if deprecated. Fixed by + // https://webrtc.googlesource.com/src/+/5b1477839d8569291b88dfe950089d0ebf34bc8f +#if 0 + auto ids = receiver->stream_ids(); + if (!ids.empty()) { + return ids[0]; + } +#else + // Use internal implementation of stream_ids() + auto streams = receiver->streams(); + if (!streams.empty()) { + return streams[0]->id(); + } +#endif + } + // Fallback on track's ID, even though it's not pairable in Unified Plan (and + // technically neither in Plan B, although this works in practice). + if (rtc::scoped_refptr track = + receiver->track()) { + return track->id(); + } + return {}; +} + webrtc::PeerConnectionInterface::IceTransportsType ICETransportTypeToNative( IceTransportType mrsValue) { using Native = webrtc::PeerConnectionInterface::IceTransportsType; @@ -1330,7 +2283,8 @@ ErrorOr> PeerConnection::create( // Ensure the factory exists RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); + rtc::scoped_refptr pc_factory = + global_factory->GetPeerConnectionFactory(); if (!pc_factory) { return Error(Result::kUnknownError); } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h index fc00aac95..a533c060f 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h @@ -6,6 +6,8 @@ #include "audio_frame_observer.h" #include "callback.h" #include "data_channel.h" +#include "media/audio_transceiver.h" +#include "media/video_transceiver.h" #include "mrs_errors.h" #include "refptr.h" #include "tracked_object.h" @@ -14,6 +16,7 @@ namespace Microsoft::MixedReality::WebRTC { class PeerConnection; +class LocalAudioTrack; class LocalVideoTrack; class ExternalVideoTrackSource; class DataChannel; @@ -34,34 +37,36 @@ struct BitrateSettings { /// The PeerConnection class is the entry point to most of WebRTC. /// It encapsulates a single connection between a local peer and a remote peer, -/// and hosts some critical events for signaling and video rendering. +/// and hosts some critical events for signaling. /// /// The high level flow to establish a connection is as follow: /// - Create a peer connection object from a factory with /// PeerConnection::create(). -/// - Register a custom callback to the various signaling events. +/// - Register custom callbacks to the various signaling events, and dispatch +/// signaling messages back and forth using the chosen signaling solution. /// - Optionally add audio/video/data tracks. These can also be added after the /// connection is established, but see remark below. -/// - Create a peer connection offer, or wait for the remote peer to send an -/// offer, and respond with an answer. +/// - Create a peer connection offer with |CreateOffer()|, or wait for the +/// remote peer to send an offer, and respond with an answer with +/// |CreateAnswer()|. /// -/// At any point, before or after the connection is initated (CreateOffer() or -/// CreateAnswer()) or established (RegisterConnectedCallback()), some audio, -/// video, and data tracks can be added to it, with the following notable +/// At any point, before or after the connection is initated (|CreateOffer()| or +/// |CreateAnswer()|) or established (|RegisterConnectedCallback()|), some +/// audio, video, and data tracks can be added to it, with the following notable /// remarks and restrictions: /// - Data tracks use the DTLS/SCTP protocol and are encrypted; this requires a /// handshake to exchange encryption secrets. This exchange is only performed /// during the initial connection handshake if at least one data track is /// present. As a consequence, at least one data track needs to be added before -/// calling CreateOffer() or CreateAnswer() if the application ever need to use -/// data channels. Otherwise trying to add a data channel after that initial +/// calling |CreateOffer()| or |CreateAnswer()| if the application ever need to +/// use data channels. Otherwise trying to add a data channel after that initial /// handshake will always fail. /// - Adding and removing any kind of tracks after the connection has been -/// initiated result in a RenegotiationNeeded event to perform a new track -/// negotitation, which requires signaling to be working. Therefore it is +/// initiated result in a |RenegotiationNeeded| event to perform a new track +/// negotiation, which requires signaling to be working. Therefore it is /// recommended when this is known in advance to add tracks before starting to -/// establish a connection, to perform the first handshake with the correct -/// tracks offer/answer right away. +/// establish a connection, in order to perform the first handshake with the +/// correct tracks offer/answer right away. class PeerConnection : public TrackedObject { public: /// Create a new PeerConnection based on the given |config|. @@ -204,47 +209,24 @@ class PeerConnection : public TrackedObject { /// been called. virtual bool IsClosed() const noexcept = 0; - // - // Remote tracks - // - - /// Callback fired when a remote track is added to the peer connection. - using TrackAddedCallback = Callback; - - /// Register a custom TrackAddedCallback. - virtual void RegisterTrackAddedCallback( - TrackAddedCallback&& callback) noexcept = 0; - - /// Callback fired when a remote track is removed from the peer connection. - using TrackRemovedCallback = Callback; - - /// Register a custom TrackRemovedCallback. - virtual void RegisterTrackRemovedCallback( - TrackRemovedCallback&& callback) noexcept = 0; - // // Video // - /// Register a custom callback invoked when a remote video frame has been - /// received and decompressed, and is ready to be displayed locally. - virtual void RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback callback) noexcept = 0; - - /// Register a custom callback invoked when a remote video frame has been - /// received and decompressed, and is ready to be displayed locally. - virtual void RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback callback) noexcept = 0; + virtual ErrorOr> AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept = 0; /// Add a video track to the peer connection. If no RTP sender/transceiver - /// exist, create a new one for that track. + /// exist, create a new one for that track. Otherwise try to reuse an existing + /// one. virtual ErrorOr> AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept = 0; + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept = 0; /// Remove a local video track from the peer connection. /// The underlying RTP sender/transceiver are kept alive but inactive. - virtual webrtc::RTCError RemoveLocalVideoTrack( + virtual Result RemoveLocalVideoTrack( LocalVideoTrack& video_track) noexcept = 0; /// Remove all tracks sharing the given video track source. @@ -253,6 +235,32 @@ class PeerConnection : public TrackedObject { virtual void RemoveLocalVideoTracksFromSource( ExternalVideoTrackSource& source) noexcept = 0; + /// Callback invoked when a remote video track is added to the peer + /// connection. + using VideoTrackAddedCallback = Callback; + + /// Register a custom |VideoTrackAddedCallback| invoked when a remote video + /// track is added to the peer connection. Only one callback can be registered + /// at a time. + virtual void RegisterVideoTrackAddedCallback( + VideoTrackAddedCallback&& callback) noexcept = 0; + + /// Callback invoked when a remote video track is removed from the peer + /// connection. + using VideoTrackRemovedCallback = Callback; + + /// Register a custom |VideoTrackRemovedCallback| invoked when a remote video + /// track is removed from the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterVideoTrackRemovedCallback( + VideoTrackRemovedCallback&& callback) noexcept = 0; + /// Rounding mode of video frame height for |SetFrameHeightRoundMode()|. /// This is only used on HoloLens 1 (UWP x86). enum class FrameHeightRoundMode { @@ -285,51 +293,47 @@ class PeerConnection : public TrackedObject { // Audio // - /// Register a custom callback invoked when a local audio frame is ready to be - /// output. - /// - /// FIXME - Current implementation of AddSink() for the local audio capture - /// device is no-op. So this callback is never fired. - virtual void RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept = 0; - - /// Register a custom callback invoked when a remote audio frame has been - /// received and uncompressed, and is ready to be output locally. - virtual void RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept = 0; - - /// Add to the peer connection an audio track backed by a local audio capture - /// device. If no RTP sender/transceiver exist, create a new one for that - /// track. - /// - /// Note: currently a single local video track is supported per peer - /// connection. - virtual bool AddLocalAudioTrack( - rtc::scoped_refptr audio_track) noexcept = 0; + virtual ErrorOr> AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept = 0; + + /// Add an audio track to the peer connection. If no RTP sender/transceiver + /// exist, create a new one for that track. Otherwise try to reuse an existing + /// one. + virtual ErrorOr> AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept = 0; - /// Remove the existing local audio track from the peer connection. + /// Remove an existing local audio track from the peer connection. /// The underlying RTP sender/transceiver are kept alive but inactive. - /// - /// Note: currently a single local audio track is supported per peer - /// connection. - virtual void RemoveLocalAudioTrack() noexcept = 0; + virtual Result RemoveLocalAudioTrack( + LocalAudioTrack& audio_track) noexcept = 0; - /// Enable or disable the local audio track. Disabled audio tracks are still - /// active but are silent, and do not consume network bandwidth. Additionally, - /// enabling/disabling the local audio track does not require an SDP exchange. - /// Therefore this is a cheaper alternative to removing and re-adding the - /// track. - /// - /// Note: currently a single local audio track is supported per peer + /// Callback invoked when a remote audio track is added to the peer /// connection. - virtual void MRS_API - SetLocalAudioTrackEnabled(bool enabled = true) noexcept = 0; + using AudioTrackAddedCallback = Callback; - /// Check if the local audio frame is enabled. - /// - /// Note: currently a single local audio track is supported per peer + /// Register a custom AudioTrackAddedCallback invoked when a remote audio + /// track is is added to the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterAudioTrackAddedCallback( + AudioTrackAddedCallback&& callback) noexcept = 0; + + /// Callback invoked when a remote audio track is removed from the peer /// connection. - virtual bool IsLocalAudioTrackEnabled() const noexcept = 0; + using AudioTrackRemovedCallback = Callback; + + /// Register a custom AudioTrackRemovedCallback invoked when a remote audio + /// track is removed from the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterAudioTrackRemovedCallback( + AudioTrackRemovedCallback&& callback) noexcept = 0; // // Data channel @@ -368,8 +372,7 @@ class PeerConnection : public TrackedObject { /// Close a given data channel and remove it from the peer connection. /// This invokes the DataChannelRemoved callback. - virtual void MRS_API - RemoveDataChannel(const DataChannel& data_channel) noexcept = 0; + virtual void RemoveDataChannel(const DataChannel& data_channel) noexcept = 0; /// Close and remove from the peer connection all data channels at once. /// This invokes the DataChannelRemoved callback for each data channel. @@ -378,8 +381,7 @@ class PeerConnection : public TrackedObject { /// Notification from a non-negotiated DataChannel that it is open, so that /// the PeerConnection can fire a DataChannelAdded event. This is called /// automatically by non-negotiated data channels; do not call manually. - virtual void MRS_API - OnDataChannelAdded(const DataChannel& data_channel) noexcept = 0; + virtual void OnDataChannelAdded(const DataChannel& data_channel) noexcept = 0; /// Internal use. void GetStats(webrtc::RTCStatsCollectorCallback* callback); @@ -394,6 +396,38 @@ class PeerConnection : public TrackedObject { virtual mrsResult RegisterInteropCallbacks( const mrsPeerConnectionInteropCallbacks& callbacks) noexcept = 0; + // + // Internal + // + + /// Internal callback on local audio track added to an audio transceiver of + /// this peer connection, to add it to the internal collection of local audio + /// tracks. + virtual void OnLocalTrackAddedToAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) = 0; + + /// Internal callback on local audio track removed from an audio transceiver + /// of this peer connection, to remove it from the internal collection of + /// local audio tracks. + virtual void OnLocalTrackRemovedFromAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) = 0; + + /// Internal callback on local video track added to a video transceiver of + /// this peer connection, to add it to the internal collection of local video + /// tracks. + virtual void OnLocalTrackAddedToVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) = 0; + + /// Internal callback on local video track removed from a video transceiver + /// of this peer connection, to remove it from the internal collection of + /// local video tracks. + virtual void OnLocalTrackRemovedFromVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) = 0; + protected: PeerConnection(RefPtr global_factory); }; diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h index 086022659..2ffb62ac2 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h @@ -19,8 +19,14 @@ class GlobalFactory; /// factory while alive. enum class ObjectType : int { kPeerConnection, + kLocalAudioTrack, kLocalVideoTrack, kExternalVideoTrackSource, + kRemoteAudioTrack, + kRemoteVideoTrack, + kDataChannel, + kAudioTransceiver, + kVideoTransceiver, }; /// Object tracked for interop, exposing helper methods for debugging purpose. diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h index d6383a3d8..d8e6b8df5 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h @@ -1,30 +1,30 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#pragma once - -#include - -inline bool IsStringNullOrEmpty(const char* str) noexcept { - return ((str == nullptr) || (str[0] == '\0')); -} - -enum class mrsShutdownOptions : uint32_t; - -inline mrsShutdownOptions operator|(mrsShutdownOptions a, - mrsShutdownOptions b) noexcept { - return (mrsShutdownOptions)((uint32_t)a | (uint32_t)b); -} - -inline mrsShutdownOptions operator&(mrsShutdownOptions a, - mrsShutdownOptions b) noexcept { - return (mrsShutdownOptions)((uint32_t)a & (uint32_t)b); -} - -inline bool operator==(mrsShutdownOptions a, uint32_t b) noexcept { - return ((uint32_t)a == b); -} - -inline bool operator!=(mrsShutdownOptions a, uint32_t b) noexcept { - return ((uint32_t)a != b); -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include + +inline bool IsStringNullOrEmpty(const char* str) noexcept { + return ((str == nullptr) || (str[0] == '\0')); +} + +enum class mrsShutdownOptions : uint32_t; + +inline mrsShutdownOptions operator|(mrsShutdownOptions a, + mrsShutdownOptions b) noexcept { + return (mrsShutdownOptions)((uint32_t)a | (uint32_t)b); +} + +inline mrsShutdownOptions operator&(mrsShutdownOptions a, + mrsShutdownOptions b) noexcept { + return (mrsShutdownOptions)((uint32_t)a & (uint32_t)b); +} + +inline bool operator==(mrsShutdownOptions a, uint32_t b) noexcept { + return ((uint32_t)a == b); +} + +inline bool operator!=(mrsShutdownOptions a, uint32_t b) noexcept { + return ((uint32_t)a != b); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj index a32d684d8..923babe5b 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj @@ -133,8 +133,20 @@ + + + + + + + + + + + + @@ -149,6 +161,18 @@ + + + + + + + + + + + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters index 4bed116fb..71860cdee 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters @@ -30,6 +30,42 @@ media + + media + + + media + + + interop + + + interop + + + interop + + + media + + + media + + + interop + + + interop + + + media + + + media + + + media + @@ -59,6 +95,42 @@ media + + media + + + media + + + media + + + interop + + + interop + + + interop + + + media + + + interop + + + interop + + + media + + + media + + + media + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj index 264582862..d93717071 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj @@ -135,7 +135,13 @@ + + + + + + @@ -148,6 +154,12 @@ + + + + + + @@ -163,6 +175,18 @@ + + + + + + + + + + + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters index 2d42823e9..a9512744f 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters @@ -30,6 +30,42 @@ media + + interop + + + interop + + + interop + + + media + + + media + + + media + + + media + + + media + + + media + + + media + + + interop + + + interop + @@ -66,6 +102,42 @@ media + + interop + + + interop + + + interop + + + media + + + media + + + media + + + media + + + media + + + media + + + media + + + interop + + + interop + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj index b6b937709..d061865d4 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj @@ -58,11 +58,13 @@ + + @@ -73,8 +75,10 @@ + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp index 30de47ab4..18d2eb59d 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp @@ -4,6 +4,9 @@ #include "pch.h" #include "audio_frame.h" +#include "interop/audio_transceiver_interop.h" +#include "interop/local_audio_track_interop.h" +#include "interop/remote_audio_track_interop.h" #include "interop_api.h" #include "test_utils.h" @@ -18,6 +21,24 @@ class AudioTrackTests : public TestUtils::TestBase {}; namespace { +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; +const mrsRemoteAudioTrackInteropHandle kFakeInteropRemoteAudioTrackHandle = + (void*)0x2; + +mrsRemoteAudioTrackInteropHandle MRS_CALL FakeIterop_RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteAudioTrackHandle; +} + +// PeerConnectionAudioTrackAddedCallback +using AudioTrackAddedCallback = + InteropCallback; + // PeerConnectionAudioFrameCallback using AudioFrameCallback = InteropCallback; @@ -80,12 +101,112 @@ bool IsSilent_int16(const int16_t* data, TEST_F(AudioTrackTests, Simple) { LocalPeerPairRaii pair; - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddLocalAudioTrack(pair.pc1())); - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // AudioTrackAdded callback. + AudioTransceiverHandle audio_transceiver2{}; + RemoteAudioTrackHandle audio_track2{}; + Event track_added2_ev; + AudioTrackAddedCallback track_added2_cb = + [&audio_track2, &audio_transceiver2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteAudioTrackHandle track_handle, + mrsAudioTransceiverInteropHandle /*interop_handle*/, + AudioTransceiverHandle transceiver_handle) { + audio_track2 = track_handle; + audio_transceiver2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterAudioTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create an audio transceiver on #1 + AudioTransceiverHandle audio_transceiver1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "transceiver1"; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &audio_transceiver1)); + ASSERT_NE(nullptr, audio_transceiver1); + + // Create the local audio track #1 + LocalAudioTrackInitConfig config{}; + LocalAudioTrackHandle audio_track1{}; + ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( + &config, "test_audio_track", &audio_track1)); + ASSERT_NE(nullptr, audio_track1); + + // Audio tracks start enabled + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the local audio track on the transceiver #1 + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetLocalTrack(audio_transceiver1, audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is audio_track1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(audio_track1, track_handle_local); + mrsLocalAudioTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, audio_track2); + ASSERT_NE(nullptr, audio_transceiver2); + + // Check transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is audio_track2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver2, &track_handle_remote)); + ASSERT_EQ(audio_track2, track_handle_remote); + mrsRemoteAudioTrackRemoveRef(track_handle_remote); + } + // Register a callback on the remote track of #2 uint32_t call_count = 0; - AudioFrameCallback audio_cb = [&call_count](const AudioFrame& frame) { + AudioFrameCallback audio2_cb = [&call_count](const AudioFrame& frame) { ASSERT_NE(nullptr, frame.data_); ASSERT_LT(0u, frame.bits_per_sample_); ASSERT_LT(0u, frame.sampling_rate_hz_); @@ -111,15 +232,12 @@ TEST_F(AudioTrackTests, Simple) { //} ++call_count; }; - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), CB(audio_cb)); - - pair.ConnectAndWait(); + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, CB(audio2_cb)); // Check several times this, because the audio "mute" is flaky, does not // really mute the audio, so check that the reported status is still // correct. - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); // Give the track some time to stream audio data Event ev; @@ -127,24 +245,111 @@ TEST_F(AudioTrackTests, Simple) { ASSERT_LT(50u, call_count) << "Expected at least 10 CPS"; // Same as above - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), nullptr, - nullptr); + // Clean-up + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); + mrsRemoteAudioTrackRemoveRef(audio_track2); + mrsAudioTransceiverRemoveRef(audio_transceiver2); + mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTransceiverRemoveRef(audio_transceiver1); } TEST_F(AudioTrackTests, Muted) { LocalPeerPairRaii pair; - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddLocalAudioTrack(pair.pc1())); + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // AudioTrackAdded callback. + AudioTransceiverHandle audio_transceiver2{}; + RemoteAudioTrackHandle audio_track2{}; + Event track_added2_ev; + AudioTrackAddedCallback track_added2_cb = + [&audio_track2, &audio_transceiver2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteAudioTrackHandle track_handle, + mrsAudioTransceiverInteropHandle /*interop_handle*/, + AudioTransceiverHandle transceiver_handle) { + audio_track2 = track_handle; + audio_transceiver2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterAudioTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create an audio transceiver on #1 + AudioTransceiverHandle audio_transceiver1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "transceiver1"; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &audio_transceiver1)); + ASSERT_NE(nullptr, audio_transceiver1); + + // Create the local audio track #1 + LocalAudioTrackInitConfig config{}; + LocalAudioTrackHandle audio_track1{}; + ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( + &config, "test_audio_track", &audio_track1)); + ASSERT_NE(nullptr, audio_track1); + + // Audio tracks start enabled + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the local audio track on the transceiver #1 + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetLocalTrack(audio_transceiver1, audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is audio_track1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(audio_track1, track_handle_local); + mrsLocalAudioTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } // Disable the audio track; it should output only silence - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionSetLocalAudioTrackEnabled( - pair.pc1(), mrsBool::kFalse)); - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(Result::kSuccess, + mrsLocalAudioTrackSetEnabled(audio_track1, mrsBool::kFalse)); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, audio_track2); + // Register a callback on the remote track of #2 uint32_t call_count = 0; AudioFrameCallback audio_cb = [&call_count, &pair](const AudioFrame& frame) { ASSERT_NE(nullptr, frame.data_); @@ -172,26 +377,26 @@ TEST_F(AudioTrackTests, Muted) { //} ++call_count; }; - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), CB(audio_cb)); - - pair.ConnectAndWait(); + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, CB(audio_cb)); // Check several times this, because the audio "mute" is flaky, does not // really mute the audio, so check that the reported status is still // correct. - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, call_count); // at least 10 CPS + ASSERT_LT(50u, call_count) << "Expected at least 10 CPS"; // Same as above - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), nullptr, - nullptr); + // Clean-up + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); + mrsRemoteAudioTrackRemoveRef(audio_track2); + mrsAudioTransceiverRemoveRef(audio_transceiver2); + mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTransceiverRemoveRef(audio_transceiver1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp new file mode 100644 index 000000000..1d7d44547 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/audio_transceiver_interop.h" +#include "interop_api.h" +#include "interop/local_audio_track_interop.h" +#include "interop/remote_audio_track_interop.h" + +#include "simple_interop.h" + +namespace { + +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteAudioTrackInteropHandle kFakeInteropRemoteAudioTrackHandle = + (void*)0x2; + +const mrsAudioTransceiverInteropHandle kFakeInteropAudioTransceiverHandle = + (void*)0x3; + +/// Fake interop callback always returning the same fake remote audio track +/// interop handle, for tests which do not care about it. +mrsRemoteAudioTrackInteropHandle MRS_CALL FakeIterop_RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteAudioTrackHandle; +} + +// PeerConnectionAudioTrackAddedCallback +using AudioTrackAddedCallback = + InteropCallback; + +// PeerConnectionI420AudioFrameCallback +using I420AudioFrameCallback = InteropCallback; + +} // namespace + +TEST(AudioTransceiver, InvalidName) { + LocalPeerPairRaii pair; + AudioTransceiverHandle transceiver_handle1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "invalid name with space"; + ASSERT_EQ(Result::kInvalidParameter, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_EQ(nullptr, transceiver_handle1); +} + +TEST(AudioTransceiver, SetDirection) { + LocalPeerPairRaii pair; + + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc1(), &interop)); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + AudioTransceiverHandle transceiver_handle1{}; + { + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "audio_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropAudioTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsAudioTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Check audio transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kInactive, dir_desired1); + + // Local audio track is NULL + LocalAudioTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote audio track is NULL + RemoteAudioTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Because the state updated event handler is registered after the transceiver + // is created, the state is stale, and applying the local description during + // |CreateOffer()| will generate some event. + ASSERT_TRUE(state_updated1_ev_local.WaitFor(10s)); + state_updated1_ev_local.Reset(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Set transceiver #1 direction to Receive + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.IsSignaled()); + state_updated1_ev_setdir.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Send+Receive + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, + dir_negotiated1); // no change + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, during + // SetRemoteDescription() + // Note: here the local description doesn't generate a state updated event + // because the local state was set with SetDirection() so is already correct. + // When the peer is creating the offer (#1), the desired direction is exactly + // the one advertized in the local description. + ASSERT_FALSE(state_updated1_ev_local.IsSignaled()); + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive because remote peer + // refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsAudioTransceiverRemoveRef(transceiver_handle1); +} + +TEST(AudioTransceiver, SetDirection_InvalidHandle) { + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsAudioTransceiverSetDirection( + nullptr, mrsTransceiverDirection::kRecvOnly)); +} + +TEST(AudioTransceiver, SetLocalTrack_InvalidHandle) { + LocalAudioTrackHandle dummy = (void*)0x1; // looks legit + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsAudioTransceiverSetLocalTrack(nullptr, dummy)); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp index f81f94c7b..39427a884 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp @@ -7,6 +7,8 @@ #include "external_video_track_source_interop.h" #include "interop_api.h" #include "local_video_track_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" #include "test_utils.h" @@ -112,6 +114,13 @@ void ValidateQuadTestFrame(const void* data, ASSERT_LE(std::fabs(err), 768.0); // +/-1 per component over 256 pixels } +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + // mrsArgb32VideoFrameCallback using Argb32VideoFrameCallback = InteropCallback; @@ -120,22 +129,114 @@ using Argb32VideoFrameCallback = InteropCallback; TEST_F(ExternalVideoTrackSourceTests, Simple) { LocalPeerPairRaii pair; - ExternalVideoTrackSourceHandle source_handle = nullptr; + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_handle) { + track_handle2 = track_handle; + transceiver_handle2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; ASSERT_EQ(mrsResult::kSuccess, mrsExternalVideoTrackSourceCreateFromArgb32Callback( - &GenerateQuadTestFrame, nullptr, &source_handle)); - ASSERT_NE(nullptr, source_handle); - mrsExternalVideoTrackSourceFinishCreation(source_handle); + &GenerateQuadTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); - LocalVideoTrackHandle track_handle = nullptr; - LocalVideoTrackFromExternalSourceInitConfig source_config{}; - ASSERT_EQ(mrsResult::kSuccess, - mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - pair.pc1(), "gen_track", source_handle, &source_config, - &track_handle)); - ASSERT_NE(nullptr, track_handle); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + // Create the local track itself for #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig source_config{}; + ASSERT_EQ(mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &source_config, "gen_track", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(mrsResult::kSuccess, + mrsPeerConnectionAddVideoTransceiver(pair.pc1(), &config, + &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Check video transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the track #1 to the transceiver #1 + ASSERT_EQ(mrsResult::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + + // Check video transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is track_handle2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle2, &track_handle_remote)); + ASSERT_EQ(track_handle2, track_handle_remote); + mrsRemoteVideoTrackRemoveRef(track_handle_remote); + } + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; Argb32VideoFrameCallback argb_cb = [&frame_count](const mrsArgb32VideoFrame& frame) { @@ -146,22 +247,22 @@ TEST_F(ExternalVideoTrackSourceTests, Simple) { frame.height_); ++frame_count; }; - mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback(pair.pc2(), - CB(argb_cb)); + mrsRemoteVideoTrackRegisterArgb32FrameCallback(track_handle2, CB(argb_cb)); - pair.ConnectAndWait(); - - // Simple timer + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsPeerConnectionRemoveLocalVideoTracksFromSource(pair.pc1(), source_handle); - mrsLocalVideoTrackRemoveRef(track_handle); - mrsExternalVideoTrackSourceShutdown(source_handle); - mrsExternalVideoTrackSourceRemoveRef(source_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterArgb32FrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); + mrsExternalVideoTrackSourceShutdown(source_handle1); + mrsExternalVideoTrackSourceRemoveRef(source_handle1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h index a0e0ff6cb..81cf9d3db 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h @@ -40,6 +40,8 @@ using namespace std::chrono_literals; #pragma warning(push) #pragma warning(disable : 4100 4127) #include "api/datachannelinterface.h" +#include "api/peerconnectioninterface.h" +#include "rtc_base/memory/aligned_malloc.h" #include "rtc_base/thread_annotations.h" #pragma warning(pop) diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h index 9cca53b73..382e2a206 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h @@ -76,6 +76,11 @@ struct Semaphore { int64_t value_{0}; }; +/// RAII helper to initialize and shutdown the library. +struct LibraryInitRaii { + // TODO - remove this +}; + /// Wrapper around an interop callback taking an extra raw pointer argument, to /// trampoline its call to a generic std::function for convenience (including /// lambda functions). diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp index 100d3f6e1..8e1f06ffe 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp @@ -32,8 +32,7 @@ TEST(PeerConnection, LocalNoIce) { pc2.handle(), type, sdp_data, &SetEventOnCompleted, &ev)); ev.Wait(); if (kOfferString == type) { - ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionCreateAnswer(pc2.handle())); + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionCreateAnswer(pc2.handle())); } }); SdpCallback sdp2_cb(pc2.handle(), [&pc1](const char* type, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp new file mode 100644 index 000000000..1d651a567 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "simple_interop.h" + +void SimpleInterop::Register(PeerConnectionHandle pc) noexcept { + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &RemoteAudioTrackCreateStatic; + interop.remote_video_track_create_object = &RemoteVideoTrackCreateStatic; + interop.data_channel_create_object = &DataChannelCreateStatic; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pc, &interop)); +} + +void SimpleInterop::Unregister(PeerConnectionHandle pc) noexcept { + mrsPeerConnectionInteropCallbacks interop{}; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pc, &interop)); +} + +void* SimpleInterop::CreateObject(ObjectType type) noexcept { + const uint32_t id = free_id_.fetch_add(1); + auto handle = std::make_unique(Handle{this, type, id}); + void* interop_handle = handle.get(); + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + objects_map_.emplace(interop_handle, std::move(handle)); + } catch (...) { + } + return interop_handle; +} + +void SimpleInterop::DestroyObject(void* obj) noexcept { + const uint32_t id = (uint32_t) reinterpret_cast(obj); + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + auto it = objects_map_.find(obj); + EXPECT_NE(objects_map_.end(), it); + if (it != objects_map_.end()) { + EXPECT_EQ(this, it->second->interop_); + EXPECT_EQ(id, it->second->id_); + objects_map_.erase(it); + } + } catch (...) { + } +} + +bool SimpleInterop::ObjectExists(ObjectType type, void* obj) noexcept { + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + auto it = objects_map_.find(obj); + if (it != objects_map_.end()) { + // If an object address is found, it must be registered with the current + // interop and with the right object type. + EXPECT_EQ(this, it->second->interop_); + EXPECT_EQ(type, it->second->object_type_); + return true; + } + } catch (...) { + } + return false; +} + +mrsRemoteAudioTrackInteropHandle SimpleInterop::RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kRemoteAudioTrack); +} + +mrsRemoteVideoTrackInteropHandle SimpleInterop::RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kRemoteVideoTrack); +} + +mrsDataChannelInteropHandle SimpleInterop::DataChannelCreate( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& /*config*/, + mrsDataChannelCallbacks* /*callbacks*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kDataChannel); +} + +mrsRemoteAudioTrackInteropHandle MRS_CALL +SimpleInterop::RemoteAudioTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + return parent_handle->interop_->RemoteAudioTrackCreate(parent, config); +} + +mrsRemoteVideoTrackInteropHandle MRS_CALL +SimpleInterop::RemoteVideoTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + return parent_handle->interop_->RemoteVideoTrackCreate(parent, config); +} + +mrsDataChannelInteropHandle MRS_CALL SimpleInterop::DataChannelCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + EXPECT_NE(nullptr, callbacks); + return parent_handle->interop_->DataChannelCreate(parent, config, callbacks); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h new file mode 100644 index 000000000..37ec4491f --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "../src/mrs_errors.h" +#include "../src/interop/global_factory.h" +#include "../include/interop_api.h" +#include "../include/peer_connection_interop.h" + +using namespace Microsoft::MixedReality::WebRTC; + +/// A simple interop layer maintaining a hash map of all objects, with assigned +/// unique identifiers. Use this to keep track of multiple interop objects in +/// tests and ensure consistency of interop handle types. +class SimpleInterop { + public: + using ObjectType = Microsoft::MixedReality::WebRTC::ObjectType; + + struct Handle { + SimpleInterop* interop_; + ObjectType object_type_; + uint32_t id_; + }; + + void Register(PeerConnectionHandle pc) noexcept; + void Unregister(PeerConnectionHandle pc) noexcept; + + void* CreateObject(ObjectType type) noexcept; + void DestroyObject(void* obj) noexcept; + bool ObjectExists(ObjectType type, void* obj) noexcept; + + mrsRemoteAudioTrackInteropHandle RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + mrsRemoteVideoTrackInteropHandle RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + mrsDataChannelInteropHandle DataChannelCreate( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept; + + static mrsRemoteAudioTrackInteropHandle MRS_CALL RemoteAudioTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + static mrsRemoteVideoTrackInteropHandle MRS_CALL RemoteVideoTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + static mrsDataChannelInteropHandle MRS_CALL + DataChannelCreateStatic(mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept; + + std::mutex objects_map_mutex_; + std::unordered_map> objects_map_; + std::atomic_uint32_t free_id_; +}; diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h index 5f35f32e6..202ad970e 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "../src/mrs_errors.h" +#include "../src/interop/global_factory.h" #include "../include/interop_api.h" #include "../include/peer_connection_interop.h" #include "../src/mrs_errors.h" @@ -16,4 +18,4 @@ mrsResult MRS_CALL MakeTestFrame(void* /*user_data*/, void CheckIsTestFrame(const I420AVideoFrame& frame); -} // namespace VideoTestUtils +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp index d81a00751..c30693b46 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp @@ -4,9 +4,12 @@ #include "pch.h" #include "external_video_track_source_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" #include "interop_api.h" #include "local_video_track_interop.h" +#include "simple_interop.h" #include "test_utils.h" #include "video_test_utils.h" @@ -14,6 +17,27 @@ namespace { +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteVideoTrackInteropHandle kFakeInteropRemoteVideoTrackHandle = + (void*)0x2; + +/// Fake interop callback always returning the same fake remote video track +/// interop handle, for tests which do not care about it. +mrsRemoteVideoTrackInteropHandle MRS_CALL FakeIterop_RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteVideoTrackHandle; +} + +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + // PeerConnectionI420VideoFrameCallback using I420VideoFrameCallback = InteropCallback; @@ -24,13 +48,129 @@ class VideoTrackTests : public TestUtils::TestBase {}; TEST_F(VideoTrackTests, Simple) { LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; - LocalVideoTrackHandle track_handle{}; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "local_video_track", - &config, &track_handle)); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + renegotiation_needed1_ev.Reset(); + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.WaitFor(1s)); + renegotiation_needed1_ev.Reset(); + } + + // Check video transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Create the local video track #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackInitConfig config{}; + ASSERT_EQ(Result::kSuccess, + mrsLocalVideoTrackCreateFromDevice(&config, "local_video_track", + &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + } + + // New tracks are enabled by default + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + + // Add the local track #1 on the transceiver #1. + ASSERT_FALSE(renegotiation_needed1_ev.IsSignaled()); + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + ASSERT_FALSE( + renegotiation_needed1_ev + .IsSignaled()); // TODO: why? because transceiver starts in SendRecv + // mode, so didn't change direction? + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + + // Check video transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is track_handle2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle2, &track_handle_remote)); + ASSERT_EQ(track_handle2, track_handle_remote); + mrsRemoteVideoTrackRemoveRef(track_handle_remote); + } + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { ASSERT_NE(nullptr, frame.ydata_); @@ -40,37 +180,91 @@ TEST_F(VideoTrackTests, Simple) { ASSERT_LT(0u, frame.height_); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsLocalVideoTrackRemoveRef(track_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); } TEST_F(VideoTrackTests, Muted) { LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; - LocalVideoTrackHandle track_handle{}; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "local_video_track", - &config, &track_handle)); + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Create the local video track #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackInitConfig config{}; + ASSERT_EQ(Result::kSuccess, + mrsLocalVideoTrackCreateFromDevice(&config, "local_video_track", + &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + } // New tracks are enabled by default - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); // Disable the video track; it should output only black frames ASSERT_EQ(Result::kSuccess, - mrsLocalVideoTrackSetEnabled(track_handle, mrsBool::kFalse)); - ASSERT_EQ(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + mrsLocalVideoTrackSetEnabled(track_handle1, mrsBool::kFalse)); + ASSERT_EQ(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + // Add the local track #1 on the transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { ASSERT_NE(nullptr, frame.ydata_); @@ -88,18 +282,20 @@ TEST_F(VideoTrackTests, Muted) { ASSERT_TRUE(all_black); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsLocalVideoTrackRemoveRef(track_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); } void MRS_CALL enumDeviceCallback(const char* id, @@ -115,7 +311,7 @@ void MRS_CALL enumDeviceCallbackCompleted(void* user_data) { } // FIXME - PeerConnection currently doesn't support multiple local video tracks -// TEST(VideoTrack, DeviceIdAll) { +// TEST_F(VideoTrackTests, DeviceIdAll) { // LocalPeerPairRaii pair; // // Event ev; @@ -133,56 +329,264 @@ void MRS_CALL enumDeviceCallbackCompleted(void* user_data) { //} TEST_F(VideoTrackTests, DeviceIdInvalid) { - LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; LocalVideoTrackHandle track_handle{}; config.video_device_id = "[[INVALID DEVICE ID]]"; - ASSERT_EQ(Result::kNotFound, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "invalid_track", - &config, &track_handle)); + ASSERT_EQ(Result::kNotFound, mrsLocalVideoTrackCreateFromDevice( + &config, "invalid_track", &track_handle)); ASSERT_EQ(nullptr, track_handle); } +TEST_F(VideoTrackTests, Multi) { + SimpleInterop simple_interop1; + SimpleInterop simple_interop2; + + mrsPeerConnectionInteropHandle h1 = + simple_interop1.CreateObject(ObjectType::kPeerConnection); + mrsPeerConnectionInteropHandle h2 = + simple_interop2.CreateObject(ObjectType::kPeerConnection); + + PeerConnectionConfiguration pc_config{}; + LocalPeerPairRaii pair(pc_config, h1, h2); + + constexpr const int kNumTracks = 5; + struct TestTrack { + int id{0}; + int frame_count{0}; + I420VideoFrameCallback frame_cb{}; + LocalVideoTrackHandle local_handle{}; + RemoteVideoTrackHandle remote_handle{}; + VideoTransceiverHandle local_transceiver_handle{}; + VideoTransceiverHandle remote_transceiver_handle{}; + }; + TestTrack tracks[kNumTracks]; + + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + simple_interop1.Register(pair.pc1()); + simple_interop2.Register(pair.pc2()); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + Semaphore track_added2_sem; + std::atomic_int32_t track_id{0}; + VideoTrackAddedCallback track_added2_cb = + [&track_added2_sem, &track_id, &tracks, kNumTracks]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_handle) { + int id = track_id.fetch_add(1); + ASSERT_LT(id, kNumTracks); + tracks[id].remote_handle = track_handle; + tracks[id].remote_transceiver_handle = transceiver_handle; + track_added2_sem.Release(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the external source for the local tracks of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create local video tracks on the local peer (#1) + LocalVideoTrackFromExternalSourceInitConfig track_config{}; + int idx = 0; + for (auto&& track : tracks) { + std::stringstream strstr; + std::string str; + VideoTransceiverInitConfig tranceiver_config{}; + strstr << "transceiver_1_" << idx; + str = strstr.str(); // keep alive + tranceiver_config.name = str.c_str(); + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &tranceiver_config, + &track.local_transceiver_handle)); + ASSERT_NE(nullptr, track.local_transceiver_handle); + strstr.clear(); + strstr << "track_1_" << idx; + str = strstr.str(); // keep alive + ASSERT_EQ(Result::kSuccess, mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &track_config, str.c_str(), + &track.local_handle)); + ASSERT_NE(nullptr, track.local_handle); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(track.local_transceiver_handle, + track.local_handle)); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track.local_handle)); + + // Check video transceiver consistency + { + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverGetLocalTrack(track.local_transceiver_handle, + &track_handle_local)); + ASSERT_EQ(track.local_handle, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverGetRemoteTrack( + track.local_transceiver_handle, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + ++idx; + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for all remote tracks to be added on #2 + ASSERT_TRUE(track_added2_sem.TryAcquireFor(5s, kNumTracks)); + for (auto&& track : tracks) { + ASSERT_NE(nullptr, track.remote_handle); + } + + // Register a frame callback for the remote video of #2 + for (auto&& track : tracks) { + track.frame_cb = [&track](const I420AVideoFrame& frame) { + ASSERT_NE(nullptr, frame.ydata_); + ASSERT_NE(nullptr, frame.udata_); + ASSERT_NE(nullptr, frame.vdata_); + ASSERT_LT(0u, frame.width_); + ASSERT_LT(0u, frame.height_); + ++track.frame_count; + }; + mrsRemoteVideoTrackRegisterI420AFrameCallback(track.remote_handle, + CB(track.frame_cb)); + } + + Event ev; + ev.WaitFor(5s); + for (auto&& track : tracks) { + ASSERT_LT(50, track.frame_count) << "Expected at least 10 FPS"; + } + + // Clean-up + for (auto&& track : tracks) { + mrsRemoteVideoTrackRegisterI420AFrameCallback(track.remote_handle, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track.remote_handle); + mrsVideoTransceiverRemoveRef(track.remote_transceiver_handle); + mrsLocalVideoTrackRemoveRef(track.local_handle); + mrsVideoTransceiverRemoveRef(track.local_transceiver_handle); + } + + simple_interop1.Unregister(pair.pc1()); + simple_interop2.Unregister(pair.pc2()); +} + TEST_F(VideoTrackTests, ExternalI420) { LocalPeerPairRaii pair; - ExternalVideoTrackSourceHandle source_handle = nullptr; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; ASSERT_EQ(mrsResult::kSuccess, mrsExternalVideoTrackSourceCreateFromI420ACallback( - &VideoTestUtils::MakeTestFrame, nullptr, &source_handle)); - ASSERT_NE(nullptr, source_handle); - mrsExternalVideoTrackSourceFinishCreation(source_handle); + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add the local track #1 on the transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); - LocalVideoTrackHandle track_handle = nullptr; - LocalVideoTrackFromExternalSourceInitConfig source_config{}; - ASSERT_EQ(mrsResult::kSuccess, - mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - pair.pc1(), "simulated_video_track", source_handle, - &source_config, &track_handle)); - ASSERT_NE(nullptr, track_handle); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { VideoTestUtils::CheckIsTestFrame(frame); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsPeerConnectionRemoveLocalVideoTrack(pair.pc1(), track_handle); - mrsLocalVideoTrackRemoveRef(track_handle); - mrsExternalVideoTrackSourceShutdown(source_handle); - mrsExternalVideoTrackSourceRemoveRef(source_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsExternalVideoTrackSourceShutdown(source_handle1); + mrsExternalVideoTrackSourceRemoveRef(source_handle1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp new file mode 100644 index 000000000..83deca12e --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp @@ -0,0 +1,612 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "external_video_track_source_interop.h" +#include "interop_api.h" +#include "local_video_track_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" + +#include "simple_interop.h" +#include "video_test_utils.h" + +namespace { + +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteVideoTrackInteropHandle kFakeInteropRemoteVideoTrackHandle = + (void*)0x2; + +const mrsVideoTransceiverInteropHandle kFakeInteropVideoTransceiverHandle = + (void*)0x3; + +/// Fake interop callback always returning the same fake remote video track +/// interop handle, for tests which do not care about it. +mrsRemoteVideoTrackInteropHandle MRS_CALL FakeIterop_RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteVideoTrackHandle; +} + +struct FakeInteropRaii { + FakeInteropRaii(std::initializer_list handles) + : handles_(handles) { + setup(); + } + ~FakeInteropRaii() { cleanup(); } + void setup() { + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = + &FakeIterop_RemoteVideoTrackCreate; + for (auto&& h : handles_) { + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(h, &interop)); + } + } + void cleanup() {} + std::vector handles_; +}; + +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + +// PeerConnectionI420VideoFrameCallback +using I420VideoFrameCallback = InteropCallback; + +} // namespace + +TEST(VideoTransceiver, InvalidName) { + LocalPeerPairRaii pair; + VideoTransceiverHandle transceiver_handle1{}; + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "invalid name with space"; + ASSERT_EQ(Result::kInvalidParameter, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_EQ(nullptr, transceiver_handle1); +} + +TEST(VideoTransceiver, SetDirection) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kInactive, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Because the state updated event handler is registered after the transceiver + // is created, the state is stale, and applying the local description during + // |CreateOffer()| will generate some event. + ASSERT_TRUE(state_updated1_ev_local.WaitFor(10s)); + state_updated1_ev_local.Reset(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Set transceiver #1 direction to Receive + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.IsSignaled()); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Send+Receive + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, + dir_negotiated1); // no change + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, during + // SetRemoteDescription() + // Note: here the local description doesn't generate a state updated event + // because the local state was set with SetDirection() so is already correct. + // When the peer is creating the offer (#1), the desired direction is exactly + // the one advertized in the local description. + ASSERT_FALSE(state_updated1_ev_local.IsSignaled()); + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive because remote peer + // refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetDirection_InvalidHandle) { + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsVideoTransceiverSetDirection( + nullptr, mrsTransceiverDirection::kRecvOnly)); +} + +TEST(VideoTransceiver, SetLocalTrackSendRecv) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Start in Send+Receive mode for this test + state_updated1_ev_setdir.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kSendRecv)); + ASSERT_TRUE(state_updated1_ev_setdir.WaitFor(10s)); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track1", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add track to transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is still Send only. + // SetLocalTrack() doesn't change the transceiver directions. + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Remove track from transceiver #1 with non-null track + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(transceiver_handle1, nullptr)); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is still Send only. + // SetLocalTrack() doesn't change the transceiver directions. + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Check video transceiver #1 consistency + //< FIXME - In theory should wait for SetRemoteDesc on #1 (from #2's answer), + // but since state doesn't change there is no way to wait for that. + { + // Again, nothing changed + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetLocalTrackRecvOnly) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Start in receive mode for this test + state_updated1_ev_setdir.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.WaitFor(10s)); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track1", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add track to transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Inactive + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Remote track from transceiver #1 with non-null track + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(transceiver_handle1, nullptr)); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Inactive + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Check video transceiver #1 consistency + //< FIXME - In theory should wait for SetRemoteDesc on #1 (from #2's answer), + // but since state doesn't change there is no way to wait for that. + { + // Note how nothing changed, because SetLocalTrack() does not change the + // desired direction of Receive, and the remote peer #2 still doesn't have a + // track to send us. + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetLocalTrack_InvalidHandle) { + LocalVideoTrackHandle dummy = (void*)0x1; // looks legit + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsVideoTransceiverSetLocalTrack(nullptr, dummy)); +}