diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h index 31df2c7ff..444e2a8dc 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/interop_api.h @@ -70,23 +70,112 @@ MRS_API void MRS_CALL mrsCloseEnum(mrsEnumHandle* handleRef) noexcept; // Interop // +struct mrsAudioTransceiverConfig; +struct mrsVideoTransceiverConfig; +struct mrsRemoteAudioTrackConfig; +struct mrsRemoteVideoTrackConfig; struct mrsDataChannelConfig; struct mrsDataChannelCallbacks; +/// Opaque handle to a native PeerConnection C++ object. +using PeerConnectionHandle = void*; + +/// Opaque handle to a native MediaTrack C++ object. +using MediaTrackHandle = void*; + +/// Opaque handle to a native AudioTransceiver C++ object. +using AudioTransceiverHandle = void*; + +/// Opaque handle to a native VideoTransceiver C++ object. +using VideoTransceiverHandle = void*; + +/// Opaque handle to a native LocalAudioTrack C++ object. +using LocalAudioTrackHandle = void*; + +/// Opaque handle to a native LocalVideoTrack C++ object. +using LocalVideoTrackHandle = void*; + +/// Opaque handle to a native RemoteAudioTrack C++ object. +using RemoteAudioTrackHandle = void*; + +/// Opaque handle to a native RemoteVideoTrack C++ object. +using RemoteVideoTrackHandle = void*; + +/// Opaque handle to a native DataChannel C++ object. +using DataChannelHandle = void*; + +/// Opaque handle to a native ExternalVideoTrackSource C++ object. +using ExternalVideoTrackSourceHandle = void*; + /// Opaque handle to the interop wrapper of a peer connection. using mrsPeerConnectionInteropHandle = void*; +/// Opaque handle to the interop wrapper of an audio transceiver. +using mrsAudioTransceiverInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a video transceiver. +using mrsVideoTransceiverInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a local audio track. +using mrsLocalAudioTrackInteropHandle = void*; + /// Opaque handle to the interop wrapper of a local video track. using mrsLocalVideoTrackInteropHandle = void*; +/// Opaque handle to the interop wrapper of a remote audio track. +using mrsRemoteAudioTrackInteropHandle = void*; + +/// Opaque handle to the interop wrapper of a remote video track. +using mrsRemoteVideoTrackInteropHandle = void*; + /// Opaque handle to the interop wrapper of a data channel. using mrsDataChannelInteropHandle = void*; +/// Callback to create an interop wrapper for an audio transceiver. +/// The callback must return the handle of the created interop wrapper. +using mrsAudioTransceiverCreateObjectCallback = + mrsAudioTransceiverInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsAudioTransceiverConfig& config) noexcept; + +/// Callback to finish the creation of the interop wrapper by assigning to it +/// the handle of the AudioTransceiver native object it wraps. +/// This is called shortly after |mrsAudioTransceiverCreateObjectCallback|, with +/// the same |mrsAudioTransceiverInteropHandle| returned by that callback. +using mrsAudioTransceiverFinishCreateCallback = + void(MRS_CALL*)(mrsAudioTransceiverInteropHandle, AudioTransceiverHandle); + +/// Callback to create an interop wrapper for a video transceiver. +/// The callback must return the handle of the created interop wrapper. +using mrsVideoTransceiverCreateObjectCallback = + mrsVideoTransceiverInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsVideoTransceiverConfig& config) noexcept; + +/// Callback to finish the creation of the interop wrapper by assigning to it +/// the handle of the VideoTransceiver native object it wraps. +/// This is called shortly after |mrsVideoTransceiverCreateObjectCallback|, with +/// the same |mrsVideoTransceiverInteropHandle| returned by that callback. +using mrsVideoTransceiverFinishCreateCallback = + void(MRS_CALL*)(mrsVideoTransceiverInteropHandle, VideoTransceiverHandle); + +/// Callback to create an interop wrapper for a remote audio track. +using mrsRemoteAudioTrackCreateObjectCallback = + mrsRemoteAudioTrackInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + +/// Callback to create an interop wrapper for a remote video track. +using mrsRemoteVideoTrackCreateObjectCallback = + mrsRemoteVideoTrackInteropHandle(MRS_CALL*)( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + /// Callback to create an interop wrapper for a data channel. using mrsDataChannelCreateObjectCallback = mrsDataChannelInteropHandle( MRS_CALL*)(mrsPeerConnectionInteropHandle parent, const mrsDataChannelConfig& config, - mrsDataChannelCallbacks* callbacks); + mrsDataChannelCallbacks* callbacks) noexcept; // // Video capture enumeration @@ -141,18 +230,6 @@ MRS_API mrsResult MRS_CALL mrsEnumVideoCaptureFormatsAsync( // Peer connection // -/// Opaque handle to a native PeerConnection C++ object. -using PeerConnectionHandle = void*; - -/// Opaque handle to a native LocalVideoTrack C++ object. -using LocalVideoTrackHandle = void*; - -/// Opaque handle to a native DataChannel C++ object. -using DataChannelHandle = void*; - -/// Opaque handle to a native ExternalVideoTrackSource C++ object. -using ExternalVideoTrackSourceHandle = void*; - /// Callback fired when the peer connection is connected, that is it finished /// the JSEP offer/answer exchange successfully. using PeerConnectionConnectedCallback = void(MRS_CALL*)(void* user_data); @@ -209,24 +286,64 @@ enum class TrackKind : uint32_t { kDataTrack = 3, }; -/// Callback fired when a remote track is added to a connection. -using PeerConnectionTrackAddedCallback = void(MRS_CALL*)(void* user_data, - TrackKind track_kind); - -/// Callback fired when a remote track is removed from a connection. -using PeerConnectionTrackRemovedCallback = - void(MRS_CALL*)(void* user_data, TrackKind track_kind); +/// Callback fired when a remote audio track is added to a connection. +/// The |audio_track| and |audio_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalAudioTrackRemoveRef()| and +/// |mrsAudioTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionAudioTrackAddedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteAudioTrackInteropHandle audio_track_wrapper, + RemoteAudioTrackHandle audio_track, + mrsAudioTransceiverInteropHandle audio_transceiver_wrapper, + AudioTransceiverHandle audio_transceiver); + +/// Callback fired when a remote audio track is removed from a connection. +/// The |audio_track| and |audio_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalAudioTrackRemoveRef()| and +/// |mrsAudioTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionAudioTrackRemovedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteAudioTrackInteropHandle audio_track_wrapper, + RemoteAudioTrackHandle audio_track, + mrsAudioTransceiverInteropHandle audio_transceiver_wrapper, + AudioTransceiverHandle audio_transceiver); + +/// Callback fired when a remote video track is added to a connection. +/// The |video_track| and |video_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalVideoTrackRemoveRef()| and +/// |mrsVideoTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionVideoTrackAddedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteVideoTrackInteropHandle video_track_wrapper, + RemoteVideoTrackHandle video_track, + mrsVideoTransceiverInteropHandle video_transceiver_wrapper, + VideoTransceiverHandle video_transceiver); + +/// Callback fired when a remote video track is removed from a connection. +/// The |video_track| and |video_transceiver| handle hold a reference to the +/// underlying native object they are associated with, and therefore must be +/// released with |mrsLocalVideoTrackRemoveRef()| and +/// |mrsVideoTransceiverRemoveRef()|, respectively, to avoid memory leaks. +using PeerConnectionVideoTrackRemovedCallback = + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, + mrsRemoteVideoTrackInteropHandle video_track_wrapper, + RemoteVideoTrackHandle video_track, + mrsVideoTransceiverInteropHandle video_transceiver_wrapper, + VideoTransceiverHandle video_transceiver); /// Callback fired when a data channel is added to the peer connection after /// being negotiated with the remote peer. using PeerConnectionDataChannelAddedCallback = - void(MRS_CALL*)(void* user_data, + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, mrsDataChannelInteropHandle data_channel_wrapper, DataChannelHandle data_channel); /// Callback fired when a data channel is remoted from the peer connection. using PeerConnectionDataChannelRemovedCallback = - void(MRS_CALL*)(void* user_data, + void(MRS_CALL*)(mrsPeerConnectionInteropHandle peer, mrsDataChannelInteropHandle data_channel_wrapper, DataChannelHandle data_channel); @@ -250,8 +367,8 @@ using mrsAudioFrame = Microsoft::MixedReality::WebRTC::AudioFrame; /// Callback fired when a local or remote (depending on use) audio frame is /// available to be consumed by the caller, usually for local output. -using PeerConnectionAudioFrameCallback = - void(MRS_CALL*)(void* user_data, const mrsAudioFrame& frame); +using mrsAudioFrameCallback = void(MRS_CALL*)(void* user_data, + const mrsAudioFrame& frame); /// Callback fired when a message is received on a data channel. using mrsDataChannelMessageCallback = void(MRS_CALL*)(void* user_data, @@ -331,37 +448,67 @@ mrsPeerConnectionCreate(PeerConnectionConfiguration config, mrsPeerConnectionInteropHandle interop_handle, PeerConnectionHandle* peerHandleOut) noexcept; +/// Callbacks needed to allow the native implementation to interact with the +/// interop layer, and in particular to react to events which necessitate +/// creating a new interop wrapper for a new native instance (whose creation was +/// not initiated by the interop, so for which the native instance is created +/// first). struct mrsPeerConnectionInteropCallbacks { + /// Construct an interop object for an AudioTransceiver instance. + mrsAudioTransceiverCreateObjectCallback audio_transceiver_create_object{}; + + /// Finish the construction of the interop object of an AudioTransceiver. + mrsAudioTransceiverFinishCreateCallback audio_transceiver_finish_create{}; + + /// Construct an interop object for a VideoTransceiver instance. + mrsVideoTransceiverCreateObjectCallback video_transceiver_create_object{}; + + /// Finish the construction of the interop object of a VideoTransceiver. + mrsVideoTransceiverFinishCreateCallback video_transceiver_finish_create{}; + + /// Construct an interop object for a RemoteAudioTrack instance. + mrsRemoteAudioTrackCreateObjectCallback remote_audio_track_create_object{}; + + /// Construct an interop object for a RemoteVideooTrack instance. + mrsRemoteVideoTrackCreateObjectCallback remote_video_track_create_object{}; + /// Construct an interop object for a DataChannel instance. mrsDataChannelCreateObjectCallback data_channel_create_object{}; }; +/// Register the interop callbacks necessary to make interop work. To +/// unregister, simply pass nullptr as the callback pointer. Only one set of +/// callbacks can be registered at a time. MRS_API mrsResult MRS_CALL mrsPeerConnectionRegisterInteropCallbacks( PeerConnectionHandle peerHandle, mrsPeerConnectionInteropCallbacks* callbacks) noexcept; -/// Register a callback fired once connected to a remote peer. -/// To unregister, simply pass nullptr as the callback pointer. +/// Register a callback invoked once connected to a remote peer. To unregister, +/// simply pass nullptr as the callback pointer. Only one callback can be +/// registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterConnectedCallback( PeerConnectionHandle peerHandle, PeerConnectionConnectedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a local message is ready to be sent via the -/// signaling service to a remote peer. +/// Register a callback invoked when a local message is ready to be sent via the +/// signaling service to a remote peer. Only one callback can be registered at a +/// time. MRS_API void MRS_CALL mrsPeerConnectionRegisterLocalSdpReadytoSendCallback( PeerConnectionHandle peerHandle, PeerConnectionLocalSdpReadytoSendCallback callback, void* user_data) noexcept; -/// Register a callback fired when an ICE candidate message is ready to be sent -/// via the signaling service to a remote peer. +/// Register a callback invoked when an ICE candidate message is ready to be +/// sent via the signaling service to a remote peer. Only one callback can be +/// registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterIceCandidateReadytoSendCallback( PeerConnectionHandle peerHandle, PeerConnectionIceCandidateReadytoSendCallback callback, void* user_data) noexcept; -/// Register a callback fired when the ICE connection state changes. +/// Register a callback invoked when the ICE connection state changes. Only one +/// callback can be registered at a time. MRS_API void MRS_CALL mrsPeerConnectionRegisterIceStateChangedCallback( PeerConnectionHandle peerHandle, PeerConnectionIceStateChangedCallback callback, @@ -374,18 +521,44 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterRenegotiationNeededCallback( PeerConnectionRenegotiationNeededCallback callback, void* user_data) noexcept; -/// Register a callback fired when a remote media track is added to the current +/// Register a callback fired when a remote audio track is added to the current /// peer connection. -MRS_API void MRS_CALL mrsPeerConnectionRegisterTrackAddedCallback( +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterAudioTrackAddedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackAddedCallback callback, + PeerConnectionAudioTrackAddedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a remote media track is removed from the +/// Register a callback fired when a remote audio track is removed from the /// current peer connection. -MRS_API void MRS_CALL mrsPeerConnectionRegisterTrackRemovedCallback( +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterAudioTrackRemovedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackRemovedCallback callback, + PeerConnectionAudioTrackRemovedCallback callback, + void* user_data) noexcept; + +/// Register a callback fired when a remote video track is added to the current +/// peer connection. +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterVideoTrackAddedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackAddedCallback callback, + void* user_data) noexcept; + +/// Register a callback fired when a remote video track is removed from the +/// current peer connection. +/// Note that the arguments include some object handles, which each hold a +/// reference to the corresponding object and therefore must be released, even +/// if the user does not make use of them in the callback. +MRS_API void MRS_CALL mrsPeerConnectionRegisterVideoTrackRemovedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackRemovedCallback callback, void* user_data) noexcept; /// Register a callback fired when a remote data channel is removed from the @@ -402,20 +575,6 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterDataChannelRemovedCallback( PeerConnectionDataChannelRemovedCallback callback, void* user_data) noexcept; -/// Register a callback fired when a video frame from a video track was received -/// from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsI420AVideoFrameCallback callback, - void* user_data) noexcept; - -/// Register a callback fired when a video frame from a video track was received -/// from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsArgb32VideoFrameCallback callback, - void* user_data) noexcept; - /// Kind of video profile. Equivalent to org::webRtc::VideoProfileKind. enum class VideoProfileKind : int32_t { kUnspecified, @@ -431,24 +590,77 @@ enum class VideoProfileKind : int32_t { kVideoHdr8, }; -/// Register a callback fired when an audio frame is available from a local -/// audio track, usually from a local audio capture device (local microphone). -/// -/// -- WARNING -- -/// Currently this callback is never fired, because the internal audio capture -/// device implementation ignores any registration and only delivers its audio -/// data to the internal WebRTC engine for sending to the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterLocalAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept; +enum class mrsTransceiverStateUpdatedReason : int32_t { + kLocalDesc, + kRemoteDesc, + kSetDirection +}; -/// Register a callback fired when an audio frame from an audio track was -/// received from the remote peer. -MRS_API void MRS_CALL mrsPeerConnectionRegisterRemoteAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept; +/// Flow direction of the media inside the transceiver. This maps to whether +/// local and/or remote tracks are attached to the transceiver. The local +/// track corresponds to the send direction, and the remote track to the +/// receive direction. +enum class mrsTransceiverDirection : int32_t { + kSendRecv = 0, + kSendOnly = 1, + kRecvOnly = 2, + kInactive = 3 +}; + +/// Same as |mrsTransceiverDirection|, but including optional unset. +enum class mrsTransceiverOptDirection : int32_t { + kNotSet = -1, + kSendRecv = 0, + kSendOnly = 1, + kRecvOnly = 2, + kInactive = 3 +}; + +/// Configuration for creating a new audio transceiver. +struct AudioTransceiverInitConfig { + /// Mandatory name of the audio transceiver. This must be a valid SDP token; + /// see |mrsSdpIsValidToken()|. + const char* name = nullptr; + + /// Initial desired direction of the transceiver media when created. + mrsTransceiverDirection desired_direction = + mrsTransceiverDirection::kSendRecv; + + /// Semi-colon separated list of stream IDs associated with the transceiver. + /// Use |Transceiver::DecodeStreamIDs()| and |Transceiver::EncodeStreamIDs()|. + const char* stream_ids = nullptr; + + /// Handle of the audio transceiver interop wrapper, if any, which will be + /// associated with the native audio transceiver object. + mrsAudioTransceiverInteropHandle transceiver_interop_handle{}; +}; + +/// Configuration for creating a new video transceiver. +struct VideoTransceiverInitConfig { + /// Mandatory name of the video transceiver. This must be a valid SDP token; + /// see |mrsSdpIsValidToken()|. + const char* name = nullptr; + + /// Initial desired direction of the transceiver media when created. + mrsTransceiverDirection desired_direction = + mrsTransceiverDirection::kSendRecv; + + /// Semi-colon separated list of stream IDs associated with the transceiver. + /// Use |Transceiver::DecodeStreamIDs()| and |Transceiver::EncodeStreamIDs()|. + const char* stream_ids = nullptr; + + /// Handle of the video transceiver interop wrapper, if any, which will be + /// associated with the native video transceiver object. + mrsVideoTransceiverInteropHandle transceiver_interop_handle{}; +}; + +/// Configuration for opening a local audio capture device and creating a local +/// audio track. +struct LocalAudioTrackInitConfig { + /// Handle of the local audio track interop wrapper, if any, which will be + /// associated with the native local audio track object. + mrsLocalAudioTrackInteropHandle track_interop_handle{}; +}; /// Configuration for opening a local video capture device and creating a local /// video track. @@ -502,19 +714,12 @@ struct LocalVideoTrackInitConfig { mrsBool enable_mrc_recording_indicator = mrsBool::kTrue; }; -/// Add a local video track from a local video capture device (webcam) to -/// the collection of tracks to send to the remote peer. -/// |video_device_id| specifies the unique identifier of a video capture -/// device to open, as obtained by enumerating devices with -/// mrsEnumVideoCaptureDevicesAsync(), or null for any device. -/// |enable_mrc| allows enabling Mixed Reality Capture on HoloLens devices, and -/// is otherwise ignored for other video capture devices. On UWP this must be -/// invoked from another thread than the main UI thread. -MRS_API mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( - PeerConnectionHandle peerHandle, - const char* track_name, - const LocalVideoTrackInitConfig* config, - LocalVideoTrackHandle* trackHandle) noexcept; +/// Configuration for creating a local video track from an external source. +struct LocalVideoTrackFromExternalSourceInitConfig { + /// Handle of the local video track interop wrapper, if any, which will be + /// associated with the native local video track object. + mrsLocalVideoTrackInteropHandle track_interop_handle{}; +}; using mrsRequestExternalI420AVideoFrameCallback = mrsResult(MRS_CALL*)(void* user_data, @@ -528,45 +733,57 @@ using mrsRequestExternalArgb32VideoFrameCallback = uint32_t request_id, int64_t timestamp_ms); -/// Configuration for creating a local video track from an external source. -struct LocalVideoTrackFromExternalSourceInitConfig { - /// Handle of the local video track interop wrapper, if any, which will be - /// associated with the native local video track object. - mrsLocalVideoTrackInteropHandle track_interop_handle{}; +/// Configuration for creating a new audio transceiver interop wrapper. +struct mrsAudioTransceiverConfig { + /// Transceiver name, for pairing with remote peer's one. This is normally + /// extracted from the receiver's stream ID #0 if the transceiver was created + /// for a remote track, or is determined by the user-provided value if created + /// with |AddAudioTransceiver()|. + const char* name{}; + + /// Zero-based media line index for the transceiver. This is the index of the + /// m= line in the SDP offer/answer as determined when adding the transceiver. + /// This is provided by the implementation and is immutable (since we don't + /// support stopping transceivers, so m= lines are not recycled). + int mline_index = -1; + + /// Initial desired direction when the transceiver is created. If the + /// transceiver is created by the implementation, this is generally set to the + /// current value on the implementation object, to keep the interop wrapper in + /// sync. + mrsTransceiverDirection initial_desired_direction = + mrsTransceiverDirection::kSendRecv; }; -/// Add a local video track from a custom video source external to the -/// implementation. This allows feeding into WebRTC frames from any source, -/// including generated or synthetic frames, for example for testing. -/// The track source initially starts as capuring. Capture can be stopped with -/// |mrsExternalVideoTrackSourceShutdown|. -/// This returns a handle to a newly allocated object, which must be released -/// once not used anymore with |mrsLocalVideoTrackRemoveRef()|. -MRS_API mrsResult MRS_CALL -mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - PeerConnectionHandle peerHandle, - const char* track_name, - ExternalVideoTrackSourceHandle source_handle, - const LocalVideoTrackFromExternalSourceInitConfig* config, - LocalVideoTrackHandle* track_handle) noexcept; - -/// Remove a local video track from the given peer connection and destroy it. -/// After this call returned, the video track handle is invalid. -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peer_handle, - LocalVideoTrackHandle track_handle) noexcept; - -/// Remove all local video tracks backed by the given video track source from -/// the given peer connection and destroy the video track source. -/// After this call returned, the video track source handle is invalid. -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTracksFromSource( - PeerConnectionHandle peer_handle, - ExternalVideoTrackSourceHandle source_handle) noexcept; - -/// Add a local audio track from a local audio capture device (microphone) to -/// the collection of tracks to send to the remote peer. -MRS_API mrsResult MRS_CALL -mrsPeerConnectionAddLocalAudioTrack(PeerConnectionHandle peerHandle) noexcept; +/// Configuration for creating a new video transceiver interop wrapper. +struct mrsVideoTransceiverConfig { + /// Transceiver name, for pairing with remote peer's one. This is normally + /// extracted from the receiver's stream ID #0 if the transceiver was created + /// for a remote track, or is determined by the user-provided value if created + /// with |AddVideoTransceiver()|. + const char* name{}; + + /// Zero-based media line index for the transceiver. This is the index of the + /// m= line in the SDP offer/answer as determined when adding the transceiver. + /// This is provided by the implementation and is immutable (since we don't + /// support stopping transceivers, so m= lines are not recycled). + int mline_index = -1; + + /// Initial desired direction when the transceiver is created. If the + /// transceiver is created by the implementation, this is generally set to the + /// current value on the implementation object, to keep the interop wrapper in + /// sync. + mrsTransceiverDirection initial_desired_direction = + mrsTransceiverDirection::kSendRecv; +}; + +struct mrsRemoteAudioTrackConfig { + const char* track_name{}; +}; + +struct mrsRemoteVideoTrackConfig { + const char* track_name{}; +}; enum class mrsDataChannelConfigFlags : uint32_t { kOrdered = 0x1, @@ -615,24 +832,10 @@ MRS_API mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( mrsDataChannelCallbacks callbacks, DataChannelHandle* dataChannelHandleOut) noexcept; -MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peerHandle, - LocalVideoTrackHandle trackHandle) noexcept; - -MRS_API void MRS_CALL mrsPeerConnectionRemoveLocalAudioTrack( - PeerConnectionHandle peerHandle) noexcept; - MRS_API mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( PeerConnectionHandle peerHandle, DataChannelHandle dataChannelHandle) noexcept; -MRS_API mrsResult MRS_CALL -mrsPeerConnectionSetLocalAudioTrackEnabled(PeerConnectionHandle peerHandle, - mrsBool enabled) noexcept; - -MRS_API mrsBool MRS_CALL mrsPeerConnectionIsLocalAudioTrackEnabled( - PeerConnectionHandle peerHandle) noexcept; - MRS_API mrsResult MRS_CALL mrsDataChannelSendMessage(DataChannelHandle dataChannelHandle, const void* data, @@ -776,6 +979,7 @@ MRS_API void MRS_CALL mrsMemCpyStride(void* dst, int32_t src_stride, int32_t elem_size, int32_t elem_count) noexcept; + // // Stats extraction. // @@ -893,4 +1097,5 @@ mrsStatsReportGetObjects(mrsStatsReportHandle report_handle, /// Release a stats report. MRS_API mrsResult MRS_CALL mrsStatsReportRemoveRef(mrsStatsReportHandle stats_report); + } // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h index 1dd1f9f5b..1e1eaf0a6 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track_interop.h @@ -19,6 +19,21 @@ mrsLocalVideoTrackAddRef(LocalVideoTrackHandle handle) noexcept; MRS_API void MRS_CALL mrsLocalVideoTrackRemoveRef(LocalVideoTrackHandle handle) noexcept; +/// Create a new local video track by opening a local video capture device +/// (webcam). +/// [UWP] This must be invoked from another thread than the main UI thread. +MRS_API mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice( + const LocalVideoTrackInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept; + +/// Create a new local video track by using an existing external video source. +MRS_API mrsResult MRS_CALL mrsLocalVideoTrackCreateFromExternalSource( + ExternalVideoTrackSourceHandle source_handle, + const LocalVideoTrackFromExternalSourceInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept; + /// Register a custom callback to be called when the local video track captured /// a frame. The captured frames is passed to the registered callback in I420 /// encoding. diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h index 4f293d9ec..5b5908823 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection_interop.h @@ -29,4 +29,18 @@ MRS_API void MRS_CALL mrsPeerConnectionRegisterIceGatheringStateChangedCallback( mrsPeerConnectionIceGatheringStateChangedCallback callback, void* user_data) noexcept; +/// Create a new audio transceiver attached to the given peer connection. +/// The audio transceiver is initially inactive. +MRS_API mrsResult MRS_CALL +mrsPeerConnectionAddAudioTransceiver(PeerConnectionHandle peer_handle, + const AudioTransceiverInitConfig* config, + AudioTransceiverHandle* handle) noexcept; + +/// Create a new video transceiver attached to the given peer connection. +/// The audio transceiver is initially inactive. +MRS_API mrsResult MRS_CALL +mrsPeerConnectionAddVideoTransceiver(PeerConnectionHandle peer_handle, + const VideoTransceiverInitConfig* config, + VideoTransceiverHandle* handle) noexcept; + } // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp new file mode 100644 index 000000000..c6d716eeb --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.cpp @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/audio_transceiver_interop.h" +#include "media/audio_transceiver.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsAudioTransceiverAddRef(AudioTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL AudioTransceiver object."; + } +} + +void MRS_CALL +mrsAudioTransceiverRemoveRef(AudioTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to remove reference from NULL AudioTransceiver object."; + } +} + +void MRS_CALL mrsAudioTransceiverRegisterStateUpdatedCallback( + AudioTransceiverHandle handle, + mrsAudioTransceiverStateUpdatedCallback callback, + void* user_data) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RegisterStateUpdatedCallback( + Transceiver::StateUpdatedCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL mrsAudioTransceiverSetDirection( + AudioTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept { + if (auto transceiver = static_cast(transceiver_handle)) { + return transceiver->SetDirection(new_direction); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsAudioTransceiverSetLocalTrack(AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle track_handle) noexcept { + if (!transceiver_handle) { + return Result::kInvalidNativeHandle; + } + auto transceiver = static_cast(transceiver_handle); + auto track = static_cast(track_handle); + return transceiver->SetLocalTrack(track); +} + +mrsResult MRS_CALL mrsAudioTransceiverGetLocalTrack( + AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetLocalTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL mrsAudioTransceiverGetRemoteTrack( + AudioTransceiverHandle transceiver_handle, + RemoteAudioTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetRemoteTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h new file mode 100644 index 000000000..d6b769088 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/audio_transceiver_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +using mrsAudioTransceiverStateUpdatedCallback = + void(MRS_CALL*)(void* user_data, + mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated_direction, + mrsTransceiverDirection desired_direction); + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTransceiverAddRef(AudioTransceiverHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTransceiverRemoveRef(AudioTransceiverHandle handle) noexcept; + +MRS_API void MRS_CALL mrsAudioTransceiverRegisterStateUpdatedCallback( + AudioTransceiverHandle handle, + mrsAudioTransceiverStateUpdatedCallback callback, + void* user_data) noexcept; + +/// Set the new desired transceiver direction. +MRS_API mrsResult MRS_CALL +mrsAudioTransceiverSetDirection(AudioTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept; + +/// Set the local audio track associated with this transceiver. This new track +/// replaces the existing one, if any. This doesn't require any SDP +/// renegotiation. +MRS_API mrsResult MRS_CALL +mrsAudioTransceiverSetLocalTrack(AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle track_handle) noexcept; + +/// Get the local audio track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsLocalAudioTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsAudioTransceiverGetLocalTrack( + AudioTransceiverHandle transceiver_handle, + LocalAudioTrackHandle* track_handle_out) noexcept; + +/// Get the remote audio track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsRemoteAudioTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsAudioTransceiverGetRemoteTrack( + AudioTransceiverHandle transceiver_handle, + RemoteAudioTrackHandle* track_handle_out) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp index 2d9782e86..3418c2a34 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/global_factory.cpp @@ -21,10 +21,20 @@ using namespace Microsoft::MixedReality::WebRTC; /// always valid, never deallocated. std::string_view ObjectTypeToString(ObjectType type) { static_assert((int)ObjectType::kPeerConnection == 0, ""); - static_assert((int)ObjectType::kLocalVideoTrack == 1, ""); - static_assert((int)ObjectType::kExternalVideoTrackSource == 2, ""); + static_assert((int)ObjectType::kLocalAudioTrack == 1, ""); + static_assert((int)ObjectType::kLocalVideoTrack == 2, ""); + static_assert((int)ObjectType::kExternalVideoTrackSource == 3, ""); + static_assert((int)ObjectType::kRemoteAudioTrack == 4, ""); + static_assert((int)ObjectType::kRemoteVideoTrack == 5, ""); + static_assert((int)ObjectType::kDataChannel == 6, ""); + static_assert((int)ObjectType::kAudioTransceiver == 7, ""); + static_assert((int)ObjectType::kVideoTransceiver == 8, ""); constexpr const std::string_view s_types[] = { - "PeerConnection", "LocalVideoTrack", "ExternalVideoTrackSource"}; + "PeerConnection", "LocalAudioTrack", + "LocalVideoTrack", "ExternalVideoTrackSource", + "RemoteAudioTrack", "RemoteVideoTrack", + "DataChannel", "AudioTransceiver", + "VideoTransceiver"}; return s_types[(int)type]; } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp index f3394155a..9816471c2 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/interop_api.cpp @@ -10,12 +10,16 @@ #include "data_channel.h" #include "external_video_track_source_interop.h" #include "interop/global_factory.h" +#include "interop/local_audio_track_interop.h" #include "interop_api.h" +#include "local_video_track_interop.h" #include "media/external_video_track_source_impl.h" +#include "media/local_audio_track.h" #include "media/local_video_track.h" #include "peer_connection.h" #include "peer_connection_interop.h" #include "sdp_utils.h" +#include "utils.h" using namespace Microsoft::MixedReality::WebRTC; @@ -630,22 +634,51 @@ void MRS_CALL mrsPeerConnectionRegisterRenegotiationNeededCallback( } } -void MRS_CALL mrsPeerConnectionRegisterTrackAddedCallback( +void MRS_CALL mrsPeerConnectionRegisterAudioTrackAddedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackAddedCallback callback, + PeerConnectionAudioTrackAddedCallback callback, void* user_data) noexcept { if (auto peer = static_cast(peerHandle)) { - peer->RegisterTrackAddedCallback(Callback{callback, user_data}); + peer->RegisterAudioTrackAddedCallback( + Callback{ + callback, user_data}); } } -void MRS_CALL mrsPeerConnectionRegisterTrackRemovedCallback( +void MRS_CALL mrsPeerConnectionRegisterAudioTrackRemovedCallback( PeerConnectionHandle peerHandle, - PeerConnectionTrackRemovedCallback callback, + PeerConnectionAudioTrackRemovedCallback callback, void* user_data) noexcept { if (auto peer = static_cast(peerHandle)) { - peer->RegisterTrackRemovedCallback( - Callback{callback, user_data}); + peer->RegisterAudioTrackRemovedCallback( + Callback{ + callback, user_data}); + } +} + +void MRS_CALL mrsPeerConnectionRegisterVideoTrackAddedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackAddedCallback callback, + void* user_data) noexcept { + if (auto peer = static_cast(peerHandle)) { + peer->RegisterVideoTrackAddedCallback( + Callback{ + callback, user_data}); + } +} + +void MRS_CALL mrsPeerConnectionRegisterVideoTrackRemovedCallback( + PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackRemovedCallback callback, + void* user_data) noexcept { + if (auto peer = static_cast(peerHandle)) { + peer->RegisterVideoTrackRemovedCallback( + Callback{ + callback, user_data}); } } @@ -671,66 +704,68 @@ void MRS_CALL mrsPeerConnectionRegisterDataChannelRemovedCallback( } } -void MRS_CALL mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsI420AVideoFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback{callback, user_data}); +mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( + const LocalAudioTrackInitConfig* config, + const char* track_name, + LocalAudioTrackHandle* track_handle_out) noexcept { + if (IsStringNullOrEmpty(track_name)) { + RTC_LOG(LS_ERROR) << "Invalid empty local audio track name."; + return Result::kInvalidParameter; } -} + if (!track_handle_out) { + RTC_LOG(LS_ERROR) << "Invalid NULL local audio track handle."; + return Result::kInvalidParameter; + } + *track_handle_out = nullptr; -void MRS_CALL mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback( - PeerConnectionHandle peerHandle, - mrsArgb32VideoFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback{callback, user_data}); + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kInvalidOperation; } -} -void MRS_CALL mrsPeerConnectionRegisterLocalAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback{callback, user_data}); + // Create the audio track source + rtc::scoped_refptr audio_source = + pc_factory->CreateAudioSource(cricket::AudioOptions()); + if (!audio_source) { + RTC_LOG(LS_ERROR) << "Failed to create local audio source."; + return Result::kUnknownError; } -} -void MRS_CALL mrsPeerConnectionRegisterRemoteAudioFrameCallback( - PeerConnectionHandle peerHandle, - PeerConnectionAudioFrameCallback callback, - void* user_data) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback{callback, user_data}); + // Create the audio track + rtc::scoped_refptr audio_track = + pc_factory->CreateAudioTrack(track_name, audio_source); + if (!audio_track) { + RTC_LOG(LS_ERROR) << "Failed to create local audio track."; + return Result::kUnknownError; } + + // Create the audio track wrapper + RefPtr track = + new LocalAudioTrack(std::move(global_factory), std::move(audio_track), + config->track_interop_handle); + *track_handle_out = track.release(); + return Result::kSuccess; } -mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( - PeerConnectionHandle peerHandle, - const char* track_name, +mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice( const LocalVideoTrackInitConfig* config, + const char* track_name, LocalVideoTrackHandle* track_handle_out) noexcept { if (IsStringNullOrEmpty(track_name)) { RTC_LOG(LS_ERROR) << "Invalid empty local video track name."; return Result::kInvalidParameter; } + if (!config) { + RTC_LOG(LS_ERROR) << "Invalid NULL local video device configuration."; + return Result::kInvalidParameter; + } if (!track_handle_out) { RTC_LOG(LS_ERROR) << "Invalid NULL local video track handle."; return Result::kInvalidParameter; } *track_handle_out = nullptr; - auto peer = static_cast(peerHandle); - if (!peer) { - RTC_LOG(LS_ERROR) << "Invalid NULL peer connection handle."; - return Result::kInvalidNativeHandle; - } RefPtr global_factory(GlobalFactory::InstancePtr()); auto pc_factory = global_factory->GetPeerConnectionFactory(); if (!pc_factory) { @@ -767,119 +802,30 @@ mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrack( SimpleMediaConstraints::MaxFrameRate(config->framerate)); } + // Create the video track source rtc::scoped_refptr video_source = pc_factory->CreateVideoSource(std::move(video_capturer), videoConstraints.get()); if (!video_source) { return Result::kUnknownError; } + + // Create the video track rtc::scoped_refptr video_track = pc_factory->CreateVideoTrack(track_name, video_source); if (!video_track) { RTC_LOG(LS_ERROR) << "Failed to create local video track."; return Result::kUnknownError; } - auto result = peer->AddLocalVideoTrack(std::move(video_track), - config->track_interop_handle); - if (result.ok()) { - RefPtr& video_track_wrapper = result.value(); - video_track_wrapper->AddRef(); // for the handle - *track_handle_out = video_track_wrapper.get(); - return Result::kSuccess; - } - RTC_LOG(LS_ERROR) << "Failed to add local video track to peer connection."; - return Result::kUnknownError; -} -mrsResult MRS_CALL mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - PeerConnectionHandle peer_handle, - const char* track_name, - ExternalVideoTrackSourceHandle source_handle, - const LocalVideoTrackFromExternalSourceInitConfig* config, - LocalVideoTrackHandle* track_handle_out) noexcept { - if (!track_handle_out) { - return Result::kInvalidParameter; - } - *track_handle_out = nullptr; - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto track_source = - static_cast(source_handle); - if (!track_source) { - return Result::kInvalidNativeHandle; - } - RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); - if (!pc_factory) { - return Result::kUnknownError; - } - std::string track_name_str; - if (track_name && (track_name[0] != '\0')) { - track_name_str = track_name; - } else { - track_name_str = "external_track"; - } - // The video track keeps a reference to the video source; let's hope this - // does not change, because this is not explicitly mentioned in the docs, - // and the video track is the only one keeping the video source alive. - rtc::scoped_refptr video_track = - pc_factory->CreateVideoTrack(track_name_str, track_source->impl()); - if (!video_track) { - return Result::kUnknownError; - } - auto result = peer->AddLocalVideoTrack(std::move(video_track), - config->track_interop_handle); - if (result.ok()) { - *track_handle_out = result.value().release(); - return Result::kSuccess; - } - RTC_LOG(LS_ERROR) << "Failed to add local video track: " - << result.error().message(); - return Result::kUnknownError; //< TODO Convert from result.error()? -} - -mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTracksFromSource( - PeerConnectionHandle peer_handle, - ExternalVideoTrackSourceHandle source_handle) noexcept { - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto source = static_cast(source_handle); - if (!source) { - return Result::kInvalidNativeHandle; - } - peer->RemoveLocalVideoTracksFromSource(*source); + // Create the video track wrapper + RefPtr track = + new LocalVideoTrack(std::move(global_factory), std::move(video_track), + config->track_interop_handle); + *track_handle_out = track.release(); return Result::kSuccess; } -mrsResult MRS_CALL -mrsPeerConnectionAddLocalAudioTrack(PeerConnectionHandle peerHandle) noexcept { - if (auto peer = static_cast(peerHandle)) { - RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); - if (!pc_factory) { - return Result::kInvalidOperation; - } - rtc::scoped_refptr audio_source = - pc_factory->CreateAudioSource(cricket::AudioOptions()); - if (!audio_source) { - return Result::kUnknownError; - } - rtc::scoped_refptr audio_track = - pc_factory->CreateAudioTrack(kLocalAudioLabel, audio_source); - if (!audio_track) { - return Result::kUnknownError; - } - return (peer->AddLocalAudioTrack(std::move(audio_track)) - ? Result::kSuccess - : Result::kUnknownError); - } - return Result::kUnknownError; -} - mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( PeerConnectionHandle peerHandle, mrsDataChannelInteropHandle dataChannelInteropHandle, @@ -916,30 +862,6 @@ mrsResult MRS_CALL mrsPeerConnectionAddDataChannel( return data_channel.error().result(); } -mrsResult MRS_CALL mrsPeerConnectionRemoveLocalVideoTrack( - PeerConnectionHandle peer_handle, - LocalVideoTrackHandle track_handle) noexcept { - auto peer = static_cast(peer_handle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - auto track = static_cast(track_handle); - if (!track) { - return Result::kInvalidNativeHandle; - } - const mrsResult res = - (peer->RemoveLocalVideoTrack(*track).ok() ? Result::kSuccess - : Result::kUnknownError); - return res; -} - -void MRS_CALL mrsPeerConnectionRemoveLocalAudioTrack( - PeerConnectionHandle peerHandle) noexcept { - if (auto peer = static_cast(peerHandle)) { - peer->RemoveLocalAudioTrack(); - } -} - mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( PeerConnectionHandle peerHandle, DataChannelHandle dataChannelHandle) noexcept { @@ -955,26 +877,6 @@ mrsResult MRS_CALL mrsPeerConnectionRemoveDataChannel( return Result::kSuccess; } -mrsResult MRS_CALL -mrsPeerConnectionSetLocalAudioTrackEnabled(PeerConnectionHandle peerHandle, - mrsBool enabled) noexcept { - auto peer = static_cast(peerHandle); - if (!peer) { - return Result::kInvalidNativeHandle; - } - peer->SetLocalAudioTrackEnabled(enabled != mrsBool::kFalse); - return Result::kSuccess; -} - -mrsBool MRS_CALL mrsPeerConnectionIsLocalAudioTrackEnabled( - PeerConnectionHandle peerHandle) noexcept { - auto peer = static_cast(peerHandle); - if (!peer) { - return mrsBool::kFalse; - } - return (peer->IsLocalAudioTrackEnabled() ? mrsBool::kTrue : mrsBool::kFalse); -} - mrsResult MRS_CALL mrsDataChannelSendMessage(DataChannelHandle dataChannelHandle, const void* data, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp new file mode 100644 index 000000000..f0ea6082b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.cpp @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/local_audio_track_interop.h" +#include "media/local_audio_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL mrsLocalAudioTrackAddRef(LocalAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL LocalAudioTrack object."; + } +} + +void MRS_CALL +mrsLocalAudioTrackRemoveRef(LocalAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "LocalAudioTrack object."; + } +} + +// mrsLocalAudioTrackCreateFromDevice -> interop_api.cpp + +void MRS_CALL +mrsLocalAudioTrackRegisterFrameCallback(LocalAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(AudioFrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsLocalAudioTrackSetEnabled(LocalAudioTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsLocalAudioTrackIsEnabled(LocalAudioTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h new file mode 100644 index 000000000..ac578af78 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_audio_track_interop.h @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsLocalAudioTrackAddRef(LocalAudioTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsLocalAudioTrackRemoveRef(LocalAudioTrackHandle handle) noexcept; + +/// Create a new local audio track by opening a local audio capture device +/// (webcam). +MRS_API mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( + const LocalAudioTrackInitConfig* config, + const char* track_name, + LocalAudioTrackHandle* track_handle_out) noexcept; + +/// Register a custom callback to be called when the local audio track captured +/// a frame. +MRS_API void MRS_CALL +mrsLocalAudioTrackRegisterFrameCallback(LocalAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a local audio track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (silent +/// audio frames). Enabling/disabling a track is a lightweight concept similar +/// to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsLocalAudioTrackSetEnabled(LocalAudioTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local audio track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsLocalAudioTrackIsEnabled(LocalAudioTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp index 95742cd02..56b077f59 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/local_video_track_interop.cpp @@ -5,8 +5,11 @@ // line, to prevent clang-format from reordering it with other headers. #include "pch.h" +#include "global_factory.h" #include "local_video_track_interop.h" +#include "media/external_video_track_source_impl.h" #include "media/local_video_track.h" +#include "utils.h" using namespace Microsoft::MixedReality::WebRTC; @@ -29,6 +32,54 @@ mrsLocalVideoTrackRemoveRef(LocalVideoTrackHandle handle) noexcept { } } +// mrsLocalVideoTrackCreateFromDevice -> interop_api.cpp + +mrsResult MRS_CALL mrsLocalVideoTrackCreateFromExternalSource( + ExternalVideoTrackSourceHandle source_handle, + const LocalVideoTrackFromExternalSourceInitConfig* config, + const char* track_name, + LocalVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + *track_handle_out = nullptr; + + auto track_source = + static_cast(source_handle); + if (!track_source) { + return Result::kInvalidNativeHandle; + } + + std::string track_name_str; + if (!IsStringNullOrEmpty(track_name)) { + track_name_str = track_name; + } else { + track_name_str = "external_track"; + } + + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kUnknownError; + } + + // The video track keeps a reference to the video source; let's hope this + // does not change, because this is not explicitly mentioned in the docs, + // and the video track is the only one keeping the video source alive. + rtc::scoped_refptr video_track = + pc_factory->CreateVideoTrack(track_name_str, track_source->impl()); + if (!video_track) { + return Result::kUnknownError; + } + + // Create the video track wrapper + RefPtr track = + new LocalVideoTrack(std::move(global_factory), std::move(video_track), + config->track_interop_handle); + *track_handle_out = track.release(); + return Result::kSuccess; +} + void MRS_CALL mrsLocalVideoTrackRegisterI420AFrameCallback( LocalVideoTrackHandle trackHandle, mrsI420AVideoFrameCallback callback, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp index 4deda7fa2..fe3ee4a24 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/peer_connection_interop.cpp @@ -38,3 +38,43 @@ void MRS_CALL mrsPeerConnectionRegisterIceGatheringStateChangedCallback( Callback{callback, user_data}); } } + +mrsResult MRS_CALL +mrsPeerConnectionAddAudioTransceiver(PeerConnectionHandle peer_handle, + const AudioTransceiverInitConfig* config, + AudioTransceiverHandle* handle) noexcept { + if (!handle || !config) { + return Result::kInvalidParameter; + } + *handle = nullptr; + if (auto peer = static_cast(peer_handle)) { + ErrorOr> audio_transceiver = + peer->AddAudioTransceiver(*config); + if (audio_transceiver.ok()) { + *handle = (AudioTransceiverHandle)audio_transceiver.value().release(); + return Result::kSuccess; + } + return audio_transceiver.error().result(); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsPeerConnectionAddVideoTransceiver(PeerConnectionHandle peer_handle, + const VideoTransceiverInitConfig* config, + VideoTransceiverHandle* handle) noexcept { + if (!handle || !config) { + return Result::kInvalidParameter; + } + *handle = nullptr; + if (auto peer = static_cast(peer_handle)) { + ErrorOr> video_transceiver = + peer->AddVideoTransceiver(*config); + if (video_transceiver.ok()) { + *handle = (AudioTransceiverHandle)video_transceiver.value().release(); + return Result::kSuccess; + } + return video_transceiver.error().result(); + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp new file mode 100644 index 000000000..85d090b93 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.cpp @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/remote_audio_track_interop.h" +#include "media/remote_audio_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsRemoteAudioTrackAddRef(RemoteAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL RemoteAudioTrack object."; + } +} + +void MRS_CALL +mrsRemoteAudioTrackRemoveRef(RemoteAudioTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "RemoteAudioTrack object."; + } +} + +void MRS_CALL +mrsRemoteAudioTrackRegisterFrameCallback(RemoteAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(AudioFrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsRemoteAudioTrackSetEnabled(RemoteAudioTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsRemoteAudioTrackIsEnabled(RemoteAudioTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h new file mode 100644 index 000000000..2ce7d20ee --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_audio_track_interop.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteAudioTrackAddRef(RemoteAudioTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteAudioTrackRemoveRef(RemoteAudioTrackHandle handle) noexcept; + +/// Register a custom callback to be called when the local audio track received +/// a frame. +MRS_API void MRS_CALL +mrsRemoteAudioTrackRegisterFrameCallback(RemoteAudioTrackHandle trackHandle, + mrsAudioFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a local audio track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (silent +/// audio frames). Enabling/disabling a track is a lightweight concept similar +/// to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsRemoteAudioTrackSetEnabled(RemoteAudioTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local audio track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsRemoteAudioTrackIsEnabled(RemoteAudioTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp new file mode 100644 index 000000000..68cf47c2c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.cpp @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/remote_video_track_interop.h" +#include "media/remote_video_track.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsRemoteVideoTrackAddRef(RemoteVideoTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL RemoteVideoTrack object."; + } +} + +void MRS_CALL +mrsRemoteVideoTrackRemoveRef(RemoteVideoTrackHandle handle) noexcept { + if (auto track = static_cast(handle)) { + track->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "RemoteVideoTrack object."; + } +} + +void MRS_CALL mrsRemoteVideoTrackRegisterI420AFrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsI420AVideoFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(I420AFrameReadyCallback{callback, user_data}); + } +} + +void MRS_CALL mrsRemoteVideoTrackRegisterArgb32FrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsArgb32VideoFrameCallback callback, + void* user_data) noexcept { + if (auto track = static_cast(trackHandle)) { + track->SetCallback(Argb32FrameReadyCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL +mrsRemoteVideoTrackSetEnabled(RemoteVideoTrackHandle track_handle, + mrsBool enabled) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return Result::kInvalidParameter; + } + track->SetEnabled(enabled != mrsBool::kFalse); + return Result::kSuccess; +} + +mrsBool MRS_CALL +mrsRemoteVideoTrackIsEnabled(RemoteVideoTrackHandle track_handle) noexcept { + auto track = static_cast(track_handle); + if (!track) { + return mrsBool::kFalse; + } + return (track->IsEnabled() ? mrsBool::kTrue : mrsBool::kFalse); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h new file mode 100644 index 000000000..21c2e61ec --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/remote_video_track_interop.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +// +// Wrapper +// + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteVideoTrackAddRef(RemoteVideoTrackHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsRemoteVideoTrackRemoveRef(RemoteVideoTrackHandle handle) noexcept; + +/// Register a custom callback to be called when the remote video track received +/// a frame. The received frames is passed to the registered callback in I420 +/// encoding. +MRS_API void MRS_CALL mrsRemoteVideoTrackRegisterI420AFrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsI420AVideoFrameCallback callback, + void* user_data) noexcept; + +/// Register a custom callback to be called when the remote video track received +/// a frame. The received frames is passed to the registered callback in ARGB32 +/// encoding. +MRS_API void MRS_CALL mrsRemoteVideoTrackRegisterArgb32FrameCallback( + RemoteVideoTrackHandle trackHandle, + mrsArgb32VideoFrameCallback callback, + void* user_data) noexcept; + +/// Enable or disable a remote video track. Enabled tracks output their media +/// content as usual. Disabled track output some void media content (black video +/// frames, silent audio frames). Enabling/disabling a track is a lightweight +/// concept similar to "mute", which does not require an SDP renegotiation. +MRS_API mrsResult MRS_CALL +mrsRemoteVideoTrackSetEnabled(RemoteVideoTrackHandle track_handle, + mrsBool enabled) noexcept; + +/// Query a local video track for its enabled status. +MRS_API mrsBool MRS_CALL +mrsRemoteVideoTrackIsEnabled(RemoteVideoTrackHandle track_handle) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp new file mode 100644 index 000000000..dd72d16a2 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.cpp @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "interop/video_transceiver_interop.h" +#include "media/video_transceiver.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsVideoTransceiverAddRef(VideoTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL VideoTransceiver object."; + } +} + +void MRS_CALL +mrsVideoTransceiverRemoveRef(VideoTransceiverHandle handle) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to remove reference from NULL VideoTransceiver object."; + } +} + +void MRS_CALL mrsVideoTransceiverRegisterStateUpdatedCallback( + VideoTransceiverHandle handle, + mrsVideoTransceiverStateUpdatedCallback callback, + void* user_data) noexcept { + if (auto transceiver = static_cast(handle)) { + transceiver->RegisterStateUpdatedCallback( + Transceiver::StateUpdatedCallback{callback, user_data}); + } +} + +mrsResult MRS_CALL mrsVideoTransceiverSetDirection( + VideoTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept { + if (auto transceiver = static_cast(transceiver_handle)) { + return transceiver->SetDirection(new_direction); + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL +mrsVideoTransceiverSetLocalTrack(VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle track_handle) noexcept { + if (!transceiver_handle) { + return Result::kInvalidNativeHandle; + } + auto transceiver = static_cast(transceiver_handle); + auto track = static_cast(track_handle); + return transceiver->SetLocalTrack(track); +} + +mrsResult MRS_CALL mrsVideoTransceiverGetLocalTrack( + VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetLocalTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} + +mrsResult MRS_CALL mrsVideoTransceiverGetRemoteTrack( + VideoTransceiverHandle transceiver_handle, + RemoteVideoTrackHandle* track_handle_out) noexcept { + if (!track_handle_out) { + return Result::kInvalidParameter; + } + if (auto transceiver = static_cast(transceiver_handle)) { + *track_handle_out = transceiver->GetRemoteTrack().release(); + return Result::kSuccess; + } + return Result::kInvalidNativeHandle; +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h new file mode 100644 index 000000000..ff0dd6bd6 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/interop/video_transceiver_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "export.h" +#include "interop_api.h" + +extern "C" { + +using mrsVideoTransceiverStateUpdatedCallback = + void(MRS_CALL*)(void* user_data, + mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated_direction, + mrsTransceiverDirection desired_direction); + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsVideoTransceiverAddRef(VideoTransceiverHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsVideoTransceiverRemoveRef(VideoTransceiverHandle handle) noexcept; + +MRS_API void MRS_CALL mrsVideoTransceiverRegisterStateUpdatedCallback( + VideoTransceiverHandle handle, + mrsVideoTransceiverStateUpdatedCallback callback, + void* user_data) noexcept; + +/// Set the new desired transceiver direction. +MRS_API mrsResult MRS_CALL +mrsVideoTransceiverSetDirection(VideoTransceiverHandle transceiver_handle, + mrsTransceiverDirection new_direction) noexcept; + +/// Set the local video track associated with this transceiver. This new +/// track replaces the existing one, if any. This doesn't require any SDP +/// renegotiation. +MRS_API mrsResult MRS_CALL +mrsVideoTransceiverSetLocalTrack(VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle track_handle) noexcept; + +/// Get the local video track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsLocalVideoTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsVideoTransceiverGetLocalTrack( + VideoTransceiverHandle transceiver_handle, + LocalVideoTrackHandle* track_handle_out) noexcept; + +/// Get the remote video track associated with this transceiver, if any. +/// The returned handle holds a reference to the video transceiver, which must +/// be released with |mrsRemoteVideoTrackRemoveRef()| once not needed anymore. +MRS_API mrsResult MRS_CALL mrsVideoTransceiverGetRemoteTrack( + VideoTransceiverHandle transceiver_handle, + RemoteVideoTrackHandle* track_handle_out) noexcept; + +} // extern "C" diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp new file mode 100644 index 000000000..61ba129d7 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.cpp @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "audio_transceiver.h" +#include "interop/global_factory.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +AudioTransceiver::AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsAudioTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), MediaKind::kAudio, owner), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +AudioTransceiver::AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsAudioTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), + MediaKind::kAudio, + owner, + transceiver), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +AudioTransceiver::~AudioTransceiver() { + // Be sure to clean-up WebRTC objects before unregistering ourself, which + // could lead to the GlobalFactory being destroyed and the WebRTC threads + // stopped. + transceiver_ = nullptr; +} + +Result AudioTransceiver::SetDirection(Direction new_direction) noexcept { + if (transceiver_) { // Unified Plan + if (new_direction == desired_direction_) { + return Result::kSuccess; + } + transceiver_->SetDirection(ToRtp(new_direction)); + } else { // Plan B + //< TODO + return Result::kUnknownError; + } + desired_direction_ = new_direction; + FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason::kSetDirection); + return Result::kSuccess; +} + +Result AudioTransceiver::SetLocalTrack( + RefPtr local_track) noexcept { + if (local_track_ == local_track) { + return Result::kSuccess; + } + Result result = Result::kSuccess; + if (transceiver_) { // Unified Plan + // We are running under the assumption that SetTrack() never changes any of + // the transceiver's directions. This is not 100% clear in the standard, so + // double-check it here in Debug, under RTC_DCHECK_IS_ON because it's + // potentially a bit expensive (proxied calls). +#if RTC_DCHECK_IS_ON + auto desired0 = transceiver_->direction(); + auto negotiated0 = transceiver_->current_direction(); +#endif + if (!transceiver_->sender()->SetTrack(local_track ? local_track->impl() + : nullptr)) { + result = Result::kInvalidOperation; + } +#if RTC_DCHECK_IS_ON + auto desired1 = transceiver_->direction(); + auto negotiated1 = transceiver_->current_direction(); + RTC_DCHECK(desired0 == desired1); + RTC_DCHECK(negotiated0 == negotiated1); +#endif + } else { // Plan B + // auto ret = owner_->ReplaceTrackPlanB(local_track_, local_track); + // if (!ret.ok()) { + // result = ret.error().result(); + //} + result = Result::kUnknownError; //< TODO + } + if (result != Result::kSuccess) { + if (local_track) { + RTC_LOG(LS_ERROR) << "Failed to set local audio track " + << local_track->GetName() << " of audio transceiver " + << GetName() << "."; + } else { + RTC_LOG(LS_ERROR) + << "Failed to clear local audio track from audio transceiver " + << GetName() << "."; + } + return result; + } + if (local_track_) { + // Detach old local track + // Keep a pointer to the track because local_track_ gets NULL'd. No need to + // keep a reference, because owner_ has one. + LocalAudioTrack* const track = local_track_.get(); + track->OnRemovedFromPeerConnection(*owner_, this, transceiver_->sender()); + owner_->OnLocalTrackRemovedFromAudioTransceiver(*this, *track); + } + local_track_ = std::move(local_track); + if (local_track_) { + // Attach new local track + local_track_->OnAddedToPeerConnection(*owner_, this, + transceiver_->sender()); + owner_->OnLocalTrackAddedToAudioTransceiver(*this, *local_track_); + } + return Result::kSuccess; +} + +void AudioTransceiver::OnLocalTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!local_track_ || (local_track_ == track)); + local_track_ = std::move(track); +} + +void AudioTransceiver::OnRemoteTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!remote_track_ || (remote_track_ == track)); + remote_track_ = std::move(track); +} + +void AudioTransceiver::OnLocalTrackRemoved(LocalAudioTrack* track) { + RTC_DCHECK_EQ(track, local_track_.get()); + local_track_ = nullptr; +} + +void AudioTransceiver::OnRemoteTrackRemoved(RemoteAudioTrack* track) { + RTC_DCHECK_EQ(track, remote_track_.get()); + remote_track_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h new file mode 100644 index 000000000..c2451228c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/audio_transceiver.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "media/local_audio_track.h" +#include "media/remote_audio_track.h" +#include "media/transceiver.h" +#include "refptr.h" + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Transceiver containing audio tracks. +class AudioTransceiver : public Transceiver { + public: + /// Constructor for Plan B. + AudioTransceiver(RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsAudioTransceiverInteropHandle interop_handle) noexcept; + + /// Constructor for Unified Plan. + AudioTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsAudioTransceiverInteropHandle interop_handle) noexcept; + + ~AudioTransceiver() override; + + Result SetDirection(Direction new_direction) noexcept; + + Result SetLocalTrack(RefPtr local_track) noexcept; + + RefPtr GetLocalTrack() noexcept { return local_track_; } + + RefPtr GetRemoteTrack() noexcept { return remote_track_; } + + int GetMlineIndex() const noexcept { return mline_index_; } + + // + // Internal + // + + void OnLocalTrackAdded(RefPtr track); + void OnRemoteTrackAdded(RefPtr track); + void OnLocalTrackRemoved(LocalAudioTrack* track); + void OnRemoteTrackRemoved(RemoteAudioTrack* track); + mrsAudioTransceiverInteropHandle GetInteropHandle() const { + return interop_handle_; + } + + protected: + RefPtr local_track_; + RefPtr remote_track_; + + int mline_index_ = -1; + + /// Transceiver name, for pairing with the remote peer. + std::string name_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsAudioTransceiverInteropHandle interop_handle_{}; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp new file mode 100644 index 000000000..86e4d3cca --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.cpp @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "local_audio_track.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +LocalAudioTrack::LocalAudioTrack( + RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), ObjectType::kLocalAudioTrack), + track_(std::move(track)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(track_); + kind_ = TrackKind::kAudioTrack; + track_->AddSink(this); //< FIXME - Implementation is no-op +} + +LocalAudioTrack::LocalAudioTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr sender, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kLocalAudioTrack, + owner), + track_(std::move(track)), + sender_(std::move(sender)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(transceiver_); + RTC_CHECK(track_); + RTC_CHECK(sender_); + kind_ = TrackKind::kAudioTrack; + transceiver_->OnLocalTrackAdded(this); + track_->AddSink(this); //< FIXME - Implementation is no-op +} + +LocalAudioTrack::~LocalAudioTrack() { + track_->RemoveSink(this); + if (owner_) { + owner_->RemoveLocalAudioTrack(*this); + } + RTC_CHECK(!transceiver_); + RTC_CHECK(!owner_); +} + +void LocalAudioTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); +} + +bool LocalAudioTrack::IsEnabled() const noexcept { + return track_->enabled(); +} + +RefPtr LocalAudioTrack::GetTransceiver() const noexcept { + return transceiver_; +} + +webrtc::AudioTrackInterface* LocalAudioTrack::impl() const { + return track_.get(); +} + +webrtc::RtpSenderInterface* LocalAudioTrack::sender() const { + return sender_.get(); +} + +void LocalAudioTrack::OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender) { + RTC_CHECK(!owner_); + RTC_CHECK(!transceiver_); + RTC_CHECK(!sender_); + RTC_CHECK(transceiver); + RTC_CHECK(sender); + owner_ = &owner; + sender_ = std::move(sender); + transceiver_ = std::move(transceiver); + transceiver_->OnLocalTrackAdded(this); +} + +void LocalAudioTrack::OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender) { + RTC_CHECK_EQ(owner_, &old_owner); + RTC_CHECK_EQ(transceiver_.get(), old_transceiver.get()); + RTC_CHECK_EQ(sender_.get(), old_sender.get()); + owner_ = nullptr; + sender_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; +} + +void LocalAudioTrack::RemoveFromPeerConnection( + webrtc::PeerConnectionInterface& peer) { + if (sender_) { + peer.RemoveTrack(sender_); + sender_ = nullptr; + owner_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; + } +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h new file mode 100644 index 000000000..b0b9e66d8 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_audio_track.h @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "callback.h" +#include "interop_api.h" +#include "media/media_track.h" +#include "refptr.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpSenderInterface; +class AudioTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class AudioTransceiver; + +/// A local audio track is a media track for a peer connection backed by a local +/// source, and transmitted to a remote peer. +/// +/// The local nature of the track implies that the local peer has control on it, +/// including enabling or disabling the track, and removing it from the peer +/// connection. This is in contrast with a remote track reflecting a track sent +/// by the remote peer, for which the local peer has limited control. +/// +/// The local audio track is backed by a local audio track source. This is +/// typically a audio capture device (e.g. webcam), but can also be a source +/// producing programmatically generated frames. The local audio track itself +/// has no knowledge about how the source produces the frames. +class LocalAudioTrack : public AudioFrameObserver, public MediaTrack { + public: + /// Constructor for a track not added to any peer connection. + LocalAudioTrack(RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept; + + /// Constructor for a track added to a peer connection. + LocalAudioTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr sender, + mrsLocalAudioTrackInteropHandle interop_handle) noexcept; + + ~LocalAudioTrack() override; + + /// Get the name of the local audio track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the audio track. An enabled track streams its content + /// from its source to the remote peer. A disabled audio track only sends + /// black frames. + void SetEnabled(bool enabled) const noexcept; + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept; + + [[nodiscard]] RefPtr GetTransceiver() const noexcept; + + // + // Advanced use + // + + [[nodiscard]] webrtc::AudioTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpSenderInterface* sender() const; + + [[nodiscard]] mrsLocalAudioTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + /// Internal callback on added to a peer connection to update the internal + /// state of the object. + void OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender); + + /// Internal callback on removed from a peer connection to update the internal + /// state of the object. + void OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender); + + void RemoveFromPeerConnection(webrtc::PeerConnectionInterface& peer); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr sender_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsLocalAudioTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp index 559308fec..2a0f467c1 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.cpp @@ -9,18 +9,42 @@ namespace Microsoft::MixedReality::WebRTC { +LocalVideoTrack::LocalVideoTrack( + RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalVideoTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), ObjectType::kLocalVideoTrack), + track_(std::move(track)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(track_); + kind_ = TrackKind::kVideoTrack; + rtc::VideoSinkWants sink_settings{}; + sink_settings.rotation_applied = true; + track_->AddOrUpdateSink(this, sink_settings); +} + LocalVideoTrack::LocalVideoTrack( RefPtr global_factory, PeerConnection& owner, + RefPtr transceiver, rtc::scoped_refptr track, rtc::scoped_refptr sender, mrsLocalVideoTrackInteropHandle interop_handle) noexcept - : TrackedObject(std::move(global_factory), ObjectType::kLocalVideoTrack), - owner_(&owner), + : MediaTrack(std::move(global_factory), + ObjectType::kLocalVideoTrack, + owner), track_(std::move(track)), sender_(std::move(sender)), - interop_handle_(interop_handle) { + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { RTC_CHECK(owner_); + RTC_CHECK(transceiver_); + RTC_CHECK(track_); + RTC_CHECK(sender_); + kind_ = TrackKind::kVideoTrack; + transceiver_->OnLocalTrackAdded(this); rtc::VideoSinkWants sink_settings{}; sink_settings.rotation_applied = true; track_->AddOrUpdateSink(this, sink_settings); @@ -31,19 +55,20 @@ LocalVideoTrack::~LocalVideoTrack() { if (owner_) { owner_->RemoveLocalVideoTrack(*this); } + RTC_CHECK(!transceiver_); RTC_CHECK(!owner_); } -std::string LocalVideoTrack::GetName() const noexcept { - return track_->id(); +void LocalVideoTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); } bool LocalVideoTrack::IsEnabled() const noexcept { return track_->enabled(); } -void LocalVideoTrack::SetEnabled(bool enabled) const noexcept { - track_->set_enabled(enabled); +RefPtr LocalVideoTrack::GetTransceiver() const noexcept { + return transceiver_; } webrtc::VideoTrackInterface* LocalVideoTrack::impl() const { @@ -54,12 +79,42 @@ webrtc::RtpSenderInterface* LocalVideoTrack::sender() const { return sender_.get(); } +void LocalVideoTrack::OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender) { + RTC_CHECK(!owner_); + RTC_CHECK(!transceiver_); + RTC_CHECK(!sender_); + RTC_CHECK(transceiver); + RTC_CHECK(sender); + owner_ = &owner; + sender_ = std::move(sender); + transceiver_ = std::move(transceiver); + transceiver_->OnLocalTrackAdded(this); +} + +void LocalVideoTrack::OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender) { + RTC_CHECK_EQ(owner_, &old_owner); + RTC_CHECK_EQ(transceiver_.get(), old_transceiver.get()); + RTC_CHECK_EQ(sender_.get(), old_sender.get()); + owner_ = nullptr; + sender_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; +} + void LocalVideoTrack::RemoveFromPeerConnection( webrtc::PeerConnectionInterface& peer) { if (sender_) { peer.RemoveTrack(sender_); sender_ = nullptr; owner_ = nullptr; + transceiver_->OnLocalTrackRemoved(this); + transceiver_ = nullptr; } } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h index 9966b5ac7..80de95cf3 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/local_video_track.h @@ -5,6 +5,8 @@ #include "callback.h" #include "interop_api.h" +#include "media/media_track.h" +#include "refptr.h" #include "tracked_object.h" #include "video_frame_observer.h" @@ -21,6 +23,7 @@ class VideoTrackInterface; namespace Microsoft::MixedReality::WebRTC { class PeerConnection; +class VideoTransceiver; /// A local video track is a media track for a peer connection backed by a local /// source, and transmitted to a remote peer. @@ -34,17 +37,25 @@ class PeerConnection; /// typically a video capture device (e.g. webcam), but can also be a source /// producing programmatically generated frames. The local video track itself /// has no knowledge about how the source produces the frames. -class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { +class LocalVideoTrack : public VideoFrameObserver, public MediaTrack { public: + /// Constructor for a track not added to any peer connection. + LocalVideoTrack(RefPtr global_factory, + rtc::scoped_refptr track, + mrsLocalVideoTrackInteropHandle interop_handle) noexcept; + + /// Constructor for a track added to a peer connection. LocalVideoTrack(RefPtr global_factory, PeerConnection& owner, + RefPtr transceiver, rtc::scoped_refptr track, rtc::scoped_refptr sender, mrsLocalVideoTrackInteropHandle interop_handle) noexcept; + ~LocalVideoTrack() override; /// Get the name of the local video track. - std::string GetName() const noexcept override; + std::string GetName() const noexcept override { return track_name_; } /// Enable or disable the video track. An enabled track streams its content /// from its source to the remote peer. A disabled video track only sends @@ -55,6 +66,8 @@ class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { /// See |SetEnabled(bool)|. [[nodiscard]] bool IsEnabled() const noexcept; + [[nodiscard]] RefPtr GetTransceiver() const noexcept; + // // Advanced use // @@ -67,20 +80,37 @@ class LocalVideoTrack : public VideoFrameObserver, public TrackedObject { return interop_handle_; } + /// Internal callback on added to a peer connection to update the internal + /// state of the object. + void OnAddedToPeerConnection( + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr sender); + + /// Internal callback on removed from a peer connection to update the internal + /// state of the object. + void OnRemovedFromPeerConnection( + PeerConnection& old_owner, + RefPtr old_transceiver, + rtc::scoped_refptr old_sender); + void RemoveFromPeerConnection(webrtc::PeerConnectionInterface& peer); private: - /// Weak reference to the PeerConnection object owning this track. - PeerConnection* owner_{}; - /// Underlying core implementation. rtc::scoped_refptr track_; /// RTP sender this track is associated with. rtc::scoped_refptr sender_; + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + /// Optional interop handle, if associated with an interop wrapper. mrsLocalVideoTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; }; } // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp new file mode 100644 index 000000000..f29c7b246 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.cpp @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "media_track.h" +#include "peer_connection.h" + +namespace Microsoft::MixedReality::WebRTC { + +MediaTrack::MediaTrack(RefPtr global_factory, + ObjectType object_type) noexcept + : TrackedObject(std::move(global_factory), object_type) {} + +MediaTrack::MediaTrack(RefPtr global_factory, + ObjectType object_type, + PeerConnection& owner) noexcept + : TrackedObject(std::move(global_factory), object_type), owner_(&owner) { + RTC_CHECK(owner_); +} + +MediaTrack::~MediaTrack() { + RTC_CHECK(!owner_); +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h new file mode 100644 index 000000000..aba26aa66 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/media_track.h @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "str.h" +#include "tracked_object.h" +#include "video_frame_observer.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpSenderInterface; +class VideoTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Base class for all audio and video tracks. +class MediaTrack : public TrackedObject { + public: + MediaTrack(RefPtr global_factory, + ObjectType object_type) noexcept; + MediaTrack(RefPtr global_factory, + ObjectType object_type, + PeerConnection& owner) noexcept; + ~MediaTrack() override; + + /// Get the kind of track. + TrackKind GetKind() const noexcept { return kind_; } + + protected: + /// Weak reference to the PeerConnection object owning this track. + PeerConnection* owner_{}; + + /// Track kind. + TrackKind kind_ = TrackKind::kUnknownTrack; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp new file mode 100644 index 000000000..d265f334b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.cpp @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "remote_audio_track.h" + +namespace Microsoft::MixedReality::WebRTC { + +RemoteAudioTrack::RemoteAudioTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteAudioTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kRemoteAudioTrack, + owner), + track_(std::move(track)), + receiver_(std::move(receiver)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(track_); + RTC_CHECK(receiver_); + RTC_CHECK(transceiver_); + kind_ = TrackKind::kAudioTrack; + transceiver_->OnRemoteTrackAdded(this); + track_->AddSink(this); +} + +RemoteAudioTrack::~RemoteAudioTrack() { + track_->RemoveSink(this); + RTC_CHECK(!owner_); +} + +webrtc::AudioTrackInterface* RemoteAudioTrack::impl() const { + return track_.get(); +} + +webrtc::RtpReceiverInterface* RemoteAudioTrack::receiver() const { + return receiver_.get(); +} + +AudioTransceiver* RemoteAudioTrack::GetTransceiver() const { + return transceiver_.get(); +} + +void RemoteAudioTrack::OnTrackRemoved(PeerConnection& owner) { + RTC_DCHECK(owner_ == &owner); + RTC_DCHECK(receiver_ != nullptr); + RTC_DCHECK(transceiver_.get() != nullptr); + owner_ = nullptr; + receiver_ = nullptr; + transceiver_->OnRemoteTrackRemoved(this); + transceiver_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h new file mode 100644 index 000000000..6ad4567bb --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_audio_track.h @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "audio_frame_observer.h" +#include "callback.h" +#include "interop_api.h" +#include "media_track.h" +#include "refptr.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpReceiverInterface; +class AudioTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class AudioTransceiver; + +/// A remote audio track is a media track for a peer connection backed by a +/// remote audio stream received from the remote peer. +/// +/// The remote nature of the track implies that the remote peer has control on +/// it, including enabling or disabling the track, and removing it from the peer +/// connection. The local peer only has limited control over the track. +class RemoteAudioTrack : public AudioFrameObserver, public MediaTrack { + public: + RemoteAudioTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteAudioTrackInteropHandle interop_handle) noexcept; + ~RemoteAudioTrack() override; + + /// Get the name of the remote audio track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the audio track. An enabled track streams its content + /// from its source to the remote peer. A disabled audio track only sends + /// empty audio data (silence). + void SetEnabled(bool enabled) const noexcept { track_->set_enabled(enabled); } + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept { return track_->enabled(); } + + // + // Advanced use + // + + [[nodiscard]] webrtc::AudioTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpReceiverInterface* receiver() const; + [[nodiscard]] AudioTransceiver* GetTransceiver() const; + + [[nodiscard]] mrsRemoteAudioTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + // Automatically called - do not use. + void OnTrackRemoved(PeerConnection& owner); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr receiver_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsRemoteAudioTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp new file mode 100644 index 000000000..b71e64c2b --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.cpp @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "remote_video_track.h" + +namespace Microsoft::MixedReality::WebRTC { + +RemoteVideoTrack::RemoteVideoTrack( + RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteVideoTrackInteropHandle interop_handle) noexcept + : MediaTrack(std::move(global_factory), + ObjectType::kRemoteVideoTrack, + owner), + track_(std::move(track)), + receiver_(std::move(receiver)), + transceiver_(std::move(transceiver)), + interop_handle_(interop_handle), + track_name_(track_->id()) { + RTC_CHECK(owner_); + RTC_CHECK(track_); + RTC_CHECK(receiver_); + RTC_CHECK(transceiver_); + kind_ = TrackKind::kVideoTrack; + transceiver_->OnRemoteTrackAdded(this); + rtc::VideoSinkWants sink_settings{}; + sink_settings.rotation_applied = true; + track_->AddOrUpdateSink(this, sink_settings); +} + +RemoteVideoTrack::~RemoteVideoTrack() { + track_->RemoveSink(this); + RTC_CHECK(!owner_); +} + +bool RemoteVideoTrack::IsEnabled() const noexcept { + return track_->enabled(); +} + +void RemoteVideoTrack::SetEnabled(bool enabled) const noexcept { + track_->set_enabled(enabled); +} + +webrtc::VideoTrackInterface* RemoteVideoTrack::impl() const { + return track_.get(); +} + +webrtc::RtpReceiverInterface* RemoteVideoTrack::receiver() const { + return receiver_.get(); +} + +VideoTransceiver* RemoteVideoTrack::GetTransceiver() const { + return transceiver_.get(); +} + +void RemoteVideoTrack::OnTrackRemoved(PeerConnection& owner) { + RTC_DCHECK(owner_ == &owner); + RTC_DCHECK(receiver_ != nullptr); + RTC_DCHECK(transceiver_.get() != nullptr); + owner_ = nullptr; + receiver_ = nullptr; + transceiver_->OnRemoteTrackRemoved(this); + transceiver_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h new file mode 100644 index 000000000..5f4c57eb1 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/remote_video_track.h @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "media_track.h" +#include "refptr.h" +#include "tracked_object.h" +#include "video_frame_observer.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpReceiverInterface; +class VideoTrackInterface; +} // namespace webrtc + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; +class VideoTransceiver; + +/// A remote video track is a media track for a peer connection backed by a +/// remote video stream received from the remote peer. +/// +/// The remote nature of the track implies that the remote peer has control on +/// it, including enabling or disabling the track, and removing it from the peer +/// connection. The local peer only has limited control over the track. +class RemoteVideoTrack : public VideoFrameObserver, public MediaTrack { + public: + RemoteVideoTrack(RefPtr global_factory, + PeerConnection& owner, + RefPtr transceiver, + rtc::scoped_refptr track, + rtc::scoped_refptr receiver, + mrsRemoteVideoTrackInteropHandle interop_handle) noexcept; + ~RemoteVideoTrack() override; + + /// Get the name of the remote video track. + std::string GetName() const noexcept override { return track_name_; } + + /// Enable or disable the video track. An enabled track streams its content + /// from its source to the remote peer. A disabled video track only sends + /// black frames. + void SetEnabled(bool enabled) const noexcept; + + /// Check if the track is enabled. + /// See |SetEnabled(bool)|. + [[nodiscard]] bool IsEnabled() const noexcept; + + // + // Advanced use + // + + [[nodiscard]] webrtc::VideoTrackInterface* impl() const; + [[nodiscard]] webrtc::RtpReceiverInterface* receiver() const; + [[nodiscard]] VideoTransceiver* GetTransceiver() const; + + [[nodiscard]] mrsRemoteVideoTrackInteropHandle GetInteropHandle() const + noexcept { + return interop_handle_; + } + + // Automatically called - do not use. + void OnTrackRemoved(PeerConnection& owner); + + private: + /// Underlying core implementation. + rtc::scoped_refptr track_; + + /// RTP sender this track is associated with. + rtc::scoped_refptr receiver_; + + /// Transceiver this track is associated with, if any. + RefPtr transceiver_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsRemoteVideoTrackInteropHandle interop_handle_{}; + + /// Cached track name, to avoid dispatching on signaling thread. + const std::string track_name_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp new file mode 100644 index 000000000..a8465aaa0 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.cpp @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "peer_connection.h" +#include "transceiver.h" +#include "utils.h" + +namespace Microsoft::MixedReality::WebRTC { + +Transceiver::Transceiver(RefPtr global_factory, + MediaKind kind, + PeerConnection& owner) noexcept + : TrackedObject(std::move(global_factory), + kind == MediaKind::kAudio ? ObjectType::kAudioTransceiver + : ObjectType::kVideoTransceiver), + owner_(&owner), + kind_(kind) { + RTC_CHECK(owner_); + //< TODO + // RTC_CHECK(owner.sdp_semantic == webrtc::SdpSemantics::kPlanB); +} + +Transceiver::Transceiver( + RefPtr global_factory, + MediaKind kind, + PeerConnection& owner, + rtc::scoped_refptr transceiver) noexcept + : TrackedObject(std::move(global_factory), + kind == MediaKind::kAudio ? ObjectType::kAudioTransceiver + : ObjectType::kVideoTransceiver), + owner_(&owner), + kind_(kind), + transceiver_(std::move(transceiver)) { + RTC_CHECK(owner_); + //< TODO + // RTC_CHECK(owner.sdp_semantic == webrtc::SdpSemantics::kUnifiedPlan); +} + +Transceiver::~Transceiver() { + // RTC_CHECK(!owner_); +} + +std::string Transceiver::GetName() const { + return "TODO"; +} + +rtc::scoped_refptr Transceiver::impl() const { + return transceiver_; +} + +webrtc::RtpTransceiverDirection Transceiver::ToRtp(Direction direction) { + using RtpDir = webrtc::RtpTransceiverDirection; + static_assert((int)Direction::kSendRecv == (int)RtpDir::kSendRecv, ""); + static_assert((int)Direction::kSendOnly == (int)RtpDir::kSendOnly, ""); + static_assert((int)Direction::kRecvOnly == (int)RtpDir::kRecvOnly, ""); + static_assert((int)Direction::kInactive == (int)RtpDir::kInactive, ""); + return (RtpDir)direction; +} + +Transceiver::Direction Transceiver::FromRtp( + webrtc::RtpTransceiverDirection rtp_direction) { + using RtpDir = webrtc::RtpTransceiverDirection; + static_assert((int)Direction::kSendRecv == (int)RtpDir::kSendRecv, ""); + static_assert((int)Direction::kSendOnly == (int)RtpDir::kSendOnly, ""); + static_assert((int)Direction::kRecvOnly == (int)RtpDir::kRecvOnly, ""); + static_assert((int)Direction::kInactive == (int)RtpDir::kInactive, ""); + return (Direction)rtp_direction; +} + +Transceiver::OptDirection Transceiver::FromRtp( + std::optional rtp_direction) { + if (rtp_direction.has_value()) { + return (OptDirection)FromRtp(rtp_direction.value()); + } + return OptDirection::kNotSet; +} + +std::vector Transceiver::DecodeStreamIDs( + const char* encoded_stream_ids) { + if (IsStringNullOrEmpty(encoded_stream_ids)) { + return {}; + } + std::vector ids; + rtc::split(encoded_stream_ids, ';', &ids); + return ids; +} + +std::string Transceiver::EncodeStreamIDs( + const std::vector& stream_ids) { + if (stream_ids.empty()) { + return {}; + } + return rtc::join(stream_ids, ';'); +} + +void Transceiver::OnSessionDescUpdated(bool remote) { + // Parse state to check for changes + bool changed = false; + if (transceiver_) { // Unified Plan + // Check negotiated direction + auto negotiated = transceiver_->current_direction(); + if (negotiated.has_value()) { + auto newValue = FromRtp(negotiated); + if (newValue != direction_) { + direction_ = newValue; + changed = true; + } + } + + // Check desired direciton + auto desired = transceiver_->direction(); + { + auto newValue = FromRtp(desired); + if (newValue != desired_direction_) { + desired_direction_ = newValue; + changed = true; + } + } + } else { + assert(false); // not called in Plan B for now... + } + + // Invoke interop callback if any + if (changed) { + FireStateUpdatedEvent(remote + ? mrsTransceiverStateUpdatedReason::kRemoteDesc + : mrsTransceiverStateUpdatedReason::kLocalDesc); + } +} + +void Transceiver::FireStateUpdatedEvent( + mrsTransceiverStateUpdatedReason reason) { + auto lock = std::scoped_lock{cb_mutex_}; + if (auto cb = state_updated_callback_) { + cb(reason, direction_, desired_direction_); + } +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h new file mode 100644 index 000000000..474342857 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/transceiver.h @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "tracked_object.h" + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class RtpTransceiverInterface; +} + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Media kind for tracks and transceivers. +enum class MediaKind : uint32_t { kAudio, kVideo }; + +/// Base class for audio and video transceivers. +class Transceiver : public TrackedObject { + public: + using Direction = mrsTransceiverDirection; + using OptDirection = mrsTransceiverOptDirection; + + /// Construct a Plan B transceiver abstraction which tries to mimic a + /// transceiver for Plan B despite the fact that this semantic doesn't have + /// any concept of transceiver. + Transceiver(RefPtr global_factory, + MediaKind kind, + PeerConnection& owner) noexcept; + + /// Construct a Unified Plan transceiver wrapper referencing an actual WebRTC + /// transceiver implementation object as defined in Unified Plan. + Transceiver( + RefPtr global_factory, + MediaKind kind, + PeerConnection& owner, + rtc::scoped_refptr transceiver) noexcept; + + ~Transceiver() override; + + std::string GetName() const override; + + /// Get the kind of transceiver. This is generally used for determining what + /// type to static_cast<> a |Transceiver| pointer to. If this is |kAudio| then + /// the object is an |AudioTransceiver| instance, and if this is |kVideo| then + /// the object is a |VideoTransceiver| instance. + MediaKind GetMediaKind() const noexcept { return kind_; } + + /// Get the current transceiver direction. + OptDirection GetDirection() const noexcept { return direction_; } + + // + // Interop callbacks + // + + /// Callback invoked when the internal state of the transceiver has + /// been updated. + using StateUpdatedCallback = + Callback; + + void RegisterStateUpdatedCallback(StateUpdatedCallback&& callback) noexcept { + auto lock = std::scoped_lock{cb_mutex_}; + state_updated_callback_ = std::move(callback); + } + + // + // Advanced + // + + [[nodiscard]] rtc::scoped_refptr impl() + const; + + /// Callback on local description updated, to check for any change in the + /// transceiver direction and update its state. + void OnSessionDescUpdated(bool remote); + + /// Fire the StateUpdated event, invoking the |state_updated_callback_| if + /// any is registered. + void FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason reason); + + [[nodiscard]] static webrtc::RtpTransceiverDirection ToRtp( + Direction direction); + [[nodiscard]] static Direction FromRtp( + webrtc::RtpTransceiverDirection rtp_direction); + [[nodiscard]] static OptDirection FromRtp( + std::optional rtp_direction); + + [[nodiscard]] static std::vector DecodeStreamIDs( + const char* encoded_stream_ids); + [[nodiscard]] static std::string EncodeStreamIDs( + const std::vector& stream_ids); + + protected: + /// Weak reference to the PeerConnection object owning this transceiver. + PeerConnection* owner_{}; + + /// Transceiver media kind. + MediaKind kind_; + + /// Current transceiver direction, as last negotiated. + /// This does not map 1:1 with the presence/absence of the local and remote + /// tracks in |AudioTransceiver| and |VideoTransceiver|, which represent the + /// state for the next negotiation, and will differ after changing tracks but + /// before renegotiating them. This does map however with the internal concept + /// of "preferred direction" |webrtc::RtpTransceiverInterface::direction()|. + OptDirection direction_ = OptDirection::kNotSet; + + /// Next desired direction, as set by user via |SetDirection()|. + Direction desired_direction_ = Direction::kInactive; + + /// If the owner PeerConnection uses Unified Plan, pointer to the actual + /// transceiver implementation object. Otherwise NULL for Plan B. + /// This is also used as a cache of which Plan is in use, to avoid querying + /// the peer connection. + rtc::scoped_refptr transceiver_; + + /// Interop callback invoked when the internal state of the transceiver has + /// been updated. + StateUpdatedCallback state_updated_callback_ RTC_GUARDED_BY(cb_mutex_); + + std::mutex cb_mutex_; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp new file mode 100644 index 000000000..a5e31bbe8 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.cpp @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "video_transceiver.h" + +namespace Microsoft::MixedReality::WebRTC { + +VideoTransceiver::VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsVideoTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), MediaKind::kVideo, owner), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +VideoTransceiver::VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsVideoTransceiverInteropHandle interop_handle) noexcept + : Transceiver(std::move(global_factory), + MediaKind::kVideo, + owner, + transceiver), + mline_index_(mline_index), + name_(std::move(name)), + interop_handle_(interop_handle) {} + +VideoTransceiver::~VideoTransceiver() { + // Be sure to clean-up WebRTC objects before unregistering ourself, which + // could lead to the GlobalFactory being destroyed and the WebRTC threads + // stopped. + transceiver_ = nullptr; +} + +Result VideoTransceiver::SetDirection(Direction new_direction) noexcept { + if (transceiver_) { // Unified Plan + if (new_direction == desired_direction_) { + return Result::kSuccess; + } + transceiver_->SetDirection(ToRtp(new_direction)); + } else { // Plan B + //< TODO + return Result::kUnknownError; + } + desired_direction_ = new_direction; + FireStateUpdatedEvent(mrsTransceiverStateUpdatedReason::kSetDirection); + return Result::kSuccess; +} + +Result VideoTransceiver::SetLocalTrack( + RefPtr local_track) noexcept { + if (local_track_ == local_track) { + return Result::kSuccess; + } + Result result = Result::kSuccess; + if (transceiver_) { // Unified Plan + // We are running under the assumption that SetTrack() never changes any of + // the transceiver's directions. This is not 100% clear in the standard, so + // double-check it here in Debug, under RTC_DCHECK_IS_ON because it's + // potentially a bit expensive (proxied calls). +#if RTC_DCHECK_IS_ON + auto desired0 = transceiver_->direction(); + auto negotiated0 = transceiver_->current_direction(); +#endif + if (!transceiver_->sender()->SetTrack(local_track ? local_track->impl() + : nullptr)) { + result = Result::kInvalidOperation; + } +#if RTC_DCHECK_IS_ON + auto desired1 = transceiver_->direction(); + auto negotiated1 = transceiver_->current_direction(); + RTC_DCHECK(desired0 == desired1); + RTC_DCHECK(negotiated0 == negotiated1); +#endif + } else { // Plan B + // auto ret = owner_->ReplaceTrackPlanB(local_track_, local_track); + // if (!ret.ok()) { + // result = ret.error().result(); + //} + result = Result::kUnknownError; //< TODO + } + if (result != Result::kSuccess) { + if (local_track) { + RTC_LOG(LS_ERROR) << "Failed to set local video track " + << local_track->GetName() << " of video transceiver " + << GetName() << "."; + } else { + RTC_LOG(LS_ERROR) + << "Failed to clear local video track from video transceiver " + << GetName() << "."; + } + return result; + } + if (local_track_) { + // Detach old local track + // Keep a pointer to the track because local_track_ gets NULL'd. No need to + // keep a reference, because owner_ has one. + LocalVideoTrack* const track = local_track_.get(); + track->OnRemovedFromPeerConnection(*owner_, this, transceiver_->sender()); + owner_->OnLocalTrackRemovedFromVideoTransceiver(*this, *track); + } + local_track_ = std::move(local_track); + if (local_track_) { + // Attach new local track + local_track_->OnAddedToPeerConnection(*owner_, this, + transceiver_->sender()); + owner_->OnLocalTrackAddedToVideoTransceiver(*this, *local_track_); + } + return Result::kSuccess; +} + +void VideoTransceiver::OnLocalTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!local_track_ || (local_track_ == track)); + local_track_ = std::move(track); +} + +void VideoTransceiver::OnRemoteTrackAdded(RefPtr track) { + // this may be called multiple times with the same track + RTC_DCHECK(!remote_track_ || (remote_track_ == track)); + remote_track_ = std::move(track); +} + +void VideoTransceiver::OnLocalTrackRemoved(LocalVideoTrack* track) { + RTC_DCHECK_EQ(track, local_track_.get()); + local_track_ = nullptr; +} + +void VideoTransceiver::OnRemoteTrackRemoved(RemoteVideoTrack* track) { + RTC_DCHECK_EQ(track, remote_track_.get()); + remote_track_ = nullptr; +} + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h new file mode 100644 index 000000000..d21011f48 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/media/video_transceiver.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "callback.h" +#include "interop_api.h" +#include "local_video_track.h" +#include "media/remote_video_track.h" +#include "media/transceiver.h" +#include "refptr.h" + +namespace Microsoft::MixedReality::WebRTC { + +class PeerConnection; + +/// Transceiver containing video tracks. +class VideoTransceiver : public Transceiver { + public: + /// Constructor for Plan B. + VideoTransceiver(RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + mrsVideoTransceiverInteropHandle interop_handle) noexcept; + + /// Constructor for Unified Plan. + VideoTransceiver( + RefPtr global_factory, + PeerConnection& owner, + int mline_index, + std::string name, + rtc::scoped_refptr transceiver, + mrsVideoTransceiverInteropHandle interop_handle) noexcept; + + ~VideoTransceiver() override; + + Result SetDirection(Direction new_direction) noexcept; + + Result SetLocalTrack(RefPtr local_track) noexcept; + + RefPtr GetLocalTrack() noexcept { return local_track_; } + + RefPtr GetRemoteTrack() noexcept { return remote_track_; } + + int GetMlineIndex() const noexcept { return mline_index_; } + + // + // Internal + // + + void OnLocalTrackAdded(RefPtr track); + void OnRemoteTrackAdded(RefPtr track); + void OnLocalTrackRemoved(LocalVideoTrack* track); + void OnRemoteTrackRemoved(RemoteVideoTrack* track); + mrsVideoTransceiverInteropHandle GetInteropHandle() const { + return interop_handle_; + } + + protected: + RefPtr local_track_; + RefPtr remote_track_; + + int mline_index_ = -1; + + /// Transceiver name, for pairing with the remote peer. + std::string name_; + + /// Optional interop handle, if associated with an interop wrapper. + mrsVideoTransceiverInteropHandle interop_handle_{}; +}; + +} // namespace Microsoft::MixedReality::WebRTC diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp index b61cc53ba..6ee7cefc0 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp @@ -7,15 +7,17 @@ #include "audio_frame_observer.h" #include "data_channel.h" +#include "interop/global_factory.h" +#include "interop_api.h" +#include "media/local_audio_track.h" #include "media/local_video_track.h" +#include "media/remote_audio_track.h" +#include "media/remote_video_track.h" #include "peer_connection.h" #include "sdp_utils.h" +#include "utils.h" #include "video_frame_observer.h" -// Internal -#include "interop/global_factory.h" -#include "interop_api.h" - #include #if defined(_M_IX86) /* x86 */ && defined(WINAPI_FAMILY) && \ @@ -146,6 +148,25 @@ Microsoft::MixedReality::WebRTC::Error ErrorFromRTCError( ResultFromRTCErrorType(error.type()), error.message()); } +class PeerConnectionImpl; + +class StreamObserver : public webrtc::ObserverInterface { + public: + StreamObserver(PeerConnectionImpl& owner, + rtc::scoped_refptr stream) + : owner_(owner), stream_(std::move(stream)) {} + + protected: + PeerConnectionImpl& owner_; + rtc::scoped_refptr stream_; + + // + // ObserverInterface + // + + void OnChanged() override; +}; + /// Implementation of PeerConnection, which also implements /// PeerConnectionObserver at the same time to simplify interaction with /// the underlying implementation object. @@ -163,9 +184,6 @@ class PeerConnectionImpl : public PeerConnection, void SetPeerImpl(rtc::scoped_refptr impl) { peer_ = std::move(impl); - remote_video_observer_.reset(new VideoFrameObserver()); - local_audio_observer_.reset(new AudioFrameObserver()); - remote_audio_observer_.reset(new AudioFrameObserver()); } void SetName(std::string_view name) { name_ = name; } @@ -228,59 +246,47 @@ class PeerConnectionImpl : public PeerConnection, void Close() noexcept override; bool IsClosed() const noexcept override; - void RegisterTrackAddedCallback( - TrackAddedCallback&& callback) noexcept override { - auto lock = std::scoped_lock{track_added_callback_mutex_}; - track_added_callback_ = std::move(callback); - } - - void RegisterTrackRemovedCallback( - TrackRemovedCallback&& callback) noexcept override { - auto lock = std::scoped_lock{track_removed_callback_mutex_}; - track_removed_callback_ = std::move(callback); - } - - void RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback callback) noexcept override { - if (remote_video_observer_) { - remote_video_observer_->SetCallback(std::move(callback)); - } - } - - void RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback callback) noexcept override { - if (remote_video_observer_) { - remote_video_observer_->SetCallback(std::move(callback)); - } - } - + ErrorOr> AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept override; ErrorOr> AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept override; - webrtc::RTCError RemoveLocalVideoTrack( - LocalVideoTrack& video_track) noexcept override; + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept override; + Result RemoveLocalVideoTrack(LocalVideoTrack& video_track) noexcept override; void RemoveLocalVideoTracksFromSource( ExternalVideoTrackSource& source) noexcept override; - void RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept override { - if (local_audio_observer_) { - local_audio_observer_->SetCallback(std::move(callback)); - } + void RegisterVideoTrackAddedCallback( + VideoTrackAddedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + video_track_added_callback_ = std::move(callback); } - void RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept override { - if (remote_audio_observer_) { - remote_audio_observer_->SetCallback(std::move(callback)); - } + void RegisterVideoTrackRemovedCallback( + VideoTrackRemovedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + video_track_removed_callback_ = std::move(callback); + } + + ErrorOr> AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept override; + ErrorOr> AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept override; + Result RemoveLocalAudioTrack(LocalAudioTrack& audio_track) noexcept override; + + void RegisterAudioTrackAddedCallback( + AudioTrackAddedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + audio_track_added_callback_ = std::move(callback); } - bool AddLocalAudioTrack(rtc::scoped_refptr - audio_track) noexcept override; - void RemoveLocalAudioTrack() noexcept override; - void SetLocalAudioTrackEnabled(bool enabled = true) noexcept override; - bool IsLocalAudioTrackEnabled() const noexcept override; + void RegisterAudioTrackRemovedCallback( + AudioTrackRemovedCallback&& callback) noexcept override { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + audio_track_removed_callback_ = std::move(callback); + } void RegisterDataChannelAddedCallback( DataChannelAddedCallback callback) noexcept override { @@ -306,11 +312,15 @@ class PeerConnectionImpl : public PeerConnection, mrsResult RegisterInteropCallbacks( const mrsPeerConnectionInteropCallbacks& callbacks) noexcept override { - // Make a full copy of all callbacks + // Make a full copy of all callbacks. Some entries might be NULL if not + // supported by the interop. interop_callbacks_ = callbacks; return Result::kSuccess; } + void OnStreamChanged( + rtc::scoped_refptr stream) noexcept; + // // PeerConnectionObserver interface // @@ -359,12 +369,28 @@ class PeerConnectionImpl : public PeerConnection, const std::vector>& streams) noexcept override; + void OnTrack(rtc::scoped_refptr + transceiver) noexcept override; + /// Callback on remote track removed. void OnRemoveTrack(rtc::scoped_refptr receiver) noexcept override; void OnLocalDescCreated(webrtc::SessionDescriptionInterface* desc) noexcept; + // + // Internal + // + + void OnLocalTrackAddedToAudioTransceiver(AudioTransceiver& transceiver, + LocalAudioTrack& track) override; + void OnLocalTrackRemovedFromAudioTransceiver(AudioTransceiver& transceiver, + LocalAudioTrack& track) override; + void OnLocalTrackAddedToVideoTransceiver(VideoTransceiver& transceiver, + LocalVideoTrack& track) override; + void OnLocalTrackRemovedFromVideoTransceiver(VideoTransceiver& transceiver, + LocalVideoTrack& track) override; + /// The underlying PC object from the core implementation. This is NULL /// after |Close()| is called. rtc::scoped_refptr peer_; @@ -417,13 +443,21 @@ class PeerConnectionImpl : public PeerConnection, RenegotiationNeededCallback renegotiation_needed_callback_ RTC_GUARDED_BY(renegotiation_needed_callback_mutex_); - /// User callback invoked when a remote audio or video track is added. - TrackAddedCallback track_added_callback_ - RTC_GUARDED_BY(track_added_callback_mutex_); + /// User callback invoked when a remote audio track is added. + AudioTrackAddedCallback audio_track_added_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); - /// User callback invoked when a remote audio or video track is removed. - TrackRemovedCallback track_removed_callback_ - RTC_GUARDED_BY(track_removed_callback_mutex_); + /// User callback invoked when a remote audio track is removed. + AudioTrackRemovedCallback audio_track_removed_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); + + /// User callback invoked when a remote video track is added. + VideoTrackAddedCallback video_track_added_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); + + /// User callback invoked when a remote video track is removed. + VideoTrackRemovedCallback video_track_removed_callback_ + RTC_GUARDED_BY(media_track_callback_mutex_); std::mutex data_channel_added_callback_mutex_; std::mutex data_channel_removed_callback_mutex_; @@ -433,18 +467,34 @@ class PeerConnectionImpl : public PeerConnection, std::mutex ice_state_changed_callback_mutex_; std::mutex ice_gathering_state_changed_callback_mutex_; std::mutex renegotiation_needed_callback_mutex_; - std::mutex track_added_callback_mutex_; - std::mutex track_removed_callback_mutex_; + std::mutex media_track_callback_mutex_; + + std::unordered_map, + rtc::scoped_refptr> + remote_streams_; + + /// Collection of all transceivers of this peer connection. + std::vector> transceivers_ RTC_GUARDED_BY(tracks_mutex_); - rtc::scoped_refptr local_audio_track_; - rtc::scoped_refptr local_audio_sender_; - std::vector> remote_streams_; + /// Collection of all local audio tracks associated with this peer connection. + std::vector> local_audio_tracks_ + RTC_GUARDED_BY(tracks_mutex_); /// Collection of all local video tracks associated with this peer connection. std::vector> local_video_tracks_ RTC_GUARDED_BY(tracks_mutex_); - /// Mutex for all collections of all tracks. + /// Collection of all remote audio tracks associated with this peer + /// connection. + std::vector> remote_audio_tracks_ + RTC_GUARDED_BY(tracks_mutex_); + + /// Collection of all remote video tracks associated with this peer + /// connection. + std::vector> remote_video_tracks_ + RTC_GUARDED_BY(tracks_mutex_); + + /// Mutex for all collections of all tracks and transceivers. rtc::CriticalSection tracks_mutex_; /// Collection of all data channels associated with this peer connection. @@ -465,11 +515,6 @@ class PeerConnectionImpl : public PeerConnection, /// Mutex for data structures related to data channels. std::mutex data_channel_mutex_; - //< TODO - Clarify lifetime of those, for now same as this PeerConnection - std::unique_ptr local_audio_observer_; - std::unique_ptr remote_audio_observer_; - std::unique_ptr remote_video_observer_; - /// Flag to indicate if SCTP was negotiated during the initial SDP handshake /// (m=application), which allows subsequently to use data channels. If this /// is false then data channels will never connnect. This is set to true if a @@ -481,8 +526,88 @@ class PeerConnectionImpl : public PeerConnection, private: PeerConnectionImpl(const PeerConnectionImpl&) = delete; PeerConnectionImpl& operator=(const PeerConnectionImpl&) = delete; + + /// Insert a new transceiver wrapper at the given media line index. + Error InsertTransceiverAtMlineIndex(int mline_index, + RefPtr transceiver); + + /// Get an existing or create a new |AudioTransceiver| wrapper for a given RTP + /// sender of a local audio track. The sender should have an RTP transceiver + /// already, and this only takes care of finding/creating a wrapper for it, so + /// should never fail as long as the sender is indeed associated with this + /// peer connection. + ErrorOr> GetOrCreateAudioTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsAudioTransceiverInteropHandle transceiver_interop_handle); + + /// Get an existing or create a new |VideoTransceiver| wrapper for a given RTP + /// sender of a local video track. The sender should have an RTP transceiver + /// already, and this only takes care of finding/creating a wrapper for it, so + /// should never fail as long as the sender is indeed associated with this + /// peer connection. + ErrorOr> GetOrCreateVideoTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsVideoTransceiverInteropHandle transceiver_interop_handle); + + /// Get an existing or create a new |AudioTransceiver| wrapper for a given RTP + /// receiver of a newly added remote audio track. The receiver should have an + /// RTP transceiver already, and this only takes care of finding/creating a + /// wrapper for it, so should never fail as long as the receiver is indeed + /// associated with this peer connection. This is only called for new remote + /// tracks, so will only create new transceiver wrappers for some of the new + /// RTP transceivers. The callback on remote description applied will use + /// |GetOrCreateTransceiver()| to create the other ones. + ErrorOr> + GetOrCreateAudioTransceiverForNewRemoteTrack( + webrtc::RtpReceiverInterface* receiver); + + /// Get an existing or create a new |VideoTransceiver| wrapper for a given RTP + /// receiver of a newly added remote video track. The sender should have an + /// RTP transceiver already, and this only takes care of finding/creating a + /// wrapper for it, so should never fail as long as the receiver is indeed + /// associated with this peer connection. This is only called for new remote + /// tracks, so will only create new transceiver wrappers for some of the new + /// RTP transceivers. The callback on remote description applied will use + /// |GetOrCreateTransceiver()| to create the other ones. + ErrorOr> + GetOrCreateVideoTransceiverForRemoteNewTrack( + webrtc::RtpReceiverInterface* receiver); + + /// Get an existing or create a new |Transceiver| instance (either audio or + /// video) wrapper for a given RTP transceiver just created as part of a local + /// or remote description applied. This is always called for new transceivers, + /// even if no remote track is added, and will create the transceiver wrappers + /// not already created by the new remote track callbacks. + ErrorOr> GetOrCreateTransceiver( + int mline_index, + webrtc::RtpTransceiverInterface* rtp_transceiver, + std::string name); + + /// Create a new audio transceiver wrapper for an exist RTP transceiver + /// missing it. + ErrorOr> CreateAudioTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver); + + /// Create a new video transceiver wrapper for an exist RTP transceiver + /// missing it. + ErrorOr> CreateVideoTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver); + + static std::string ExtractTransceiverNameFromSender( + webrtc::RtpSenderInterface* sender); + + static std::string ExtractTransceiverNameFromReceiver( + webrtc::RtpReceiverInterface* receiver); }; +void StreamObserver::OnChanged() { + owner_.OnStreamChanged(stream_); +} + class CreateSessionDescObserver : public webrtc::CreateSessionDescriptionObserver { public: @@ -522,8 +647,9 @@ class SessionDescObserver : public webrtc::SetSessionDescriptionObserver { SessionDescObserver(Closure&& callback) : callback_(std::forward(callback)) {} void OnSuccess() override { - if (callback_) + if (callback_) { callback_(); + } } void OnFailure(webrtc::RTCError error) override { RTC_LOG(LS_ERROR) << "Error setting session description: " @@ -598,28 +724,104 @@ IceGatheringState IceGatheringStateFromImpl( return (IceGatheringState)impl_state; } +ErrorOr> PeerConnectionImpl::AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept { + if (IsClosed()) { + return Error(Result::kInvalidOperation, "The peer connection is closed."); + } + + std::string name; + if (!IsStringNullOrEmpty(config.name)) { + name = config.name; + } else { + name = rtc::CreateRandomUuid(); + } + if (!SdpIsValidToken(name)) { + rtc::StringBuilder str("Invalid video transceiver name: "); + str << name; + return Error(Result::kInvalidParameter, str.Release()); + } + + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + // Plan B doesn't have transceivers; just create a wrapper. + mline_index = (int)transceivers_.size(); // append + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), + config.transceiver_interop_handle); + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the low-level implementation object + webrtc::RtpTransceiverInit init{}; + init.direction = Transceiver::ToRtp(config.desired_direction); + init.stream_ids = Transceiver::DecodeStreamIDs(config.stream_ids); + if (!name.empty()) { + // Prepend transceiver name as first stream ID for track pairing + init.stream_ids.insert(init.stream_ids.begin(), name); + } + webrtc::RTCErrorOr> + ret = + peer_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, init); + if (!ret.ok()) { + return ErrorFromRTCError(ret.MoveError()); + } + rtc::scoped_refptr impl(ret.MoveValue()); + + // Find the mline index from the position inside the transceiver list + { + auto transceivers = peer_->GetTransceivers(); + auto it_tr = + std::find_if(transceivers.begin(), transceivers.end(), + [&impl](auto const& tr) { return (tr == impl); }); + RTC_DCHECK(it_tr != transceivers.end()); + mline_index = (int)std::distance(transceivers.begin(), it_tr); + } + + // Create the transceiver wrapper + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + config.transceiver_interop_handle); + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic."); + } + RTC_DCHECK(wrapper); + InsertTransceiverAtMlineIndex(mline_index, wrapper); + return wrapper; +} + ErrorOr> PeerConnectionImpl::AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept { + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept { if (IsClosed()) { - return Microsoft::MixedReality::WebRTC::Error( - Result::kInvalidOperation, "The peer connection is closed."); + return Error(Result::kInvalidOperation, "The peer connection is closed."); } auto result = peer_->AddTrack(video_track, {kAudioVideoStreamId}); - if (result.ok()) { - RefPtr track = - new LocalVideoTrack(global_factory_, *this, std::move(video_track), - std::move(result.MoveValue()), interop_handle); - { - rtc::CritScope lock(&tracks_mutex_); - local_video_tracks_.push_back(track); - } - return track; + if (!result.ok()) { + return ErrorFromRTCError(result.MoveError()); + } + rtc::scoped_refptr sender = result.MoveValue(); + ErrorOr> ret = + GetOrCreateVideoTransceiverForSender(sender, transceiver_interop_handle); + if (!ret.ok()) { + peer_->RemoveTrack(sender); + return ret.MoveError(); + } + RefPtr transceiver = ret.MoveValue(); + RefPtr track = new LocalVideoTrack( + global_factory_, *this, std::move(transceiver), std::move(video_track), + std::move(sender), track_interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + local_video_tracks_.push_back(track); } - return ErrorFromRTCError(result.MoveError()); + return track; } -webrtc::RTCError PeerConnectionImpl::RemoveLocalVideoTrack( +Result PeerConnectionImpl::RemoveLocalVideoTrack( LocalVideoTrack& video_track) noexcept { rtc::CritScope lock(&tracks_mutex_); auto it = std::find_if(local_video_tracks_.begin(), local_video_tracks_.end(), @@ -627,15 +829,13 @@ webrtc::RTCError PeerConnectionImpl::RemoveLocalVideoTrack( return track.get() == &video_track; }); if (it == local_video_tracks_.end()) { - return webrtc::RTCError( - webrtc::RTCErrorType::INVALID_PARAMETER, - "The video track is not associated with the peer connection."); + return Result::kInvalidParameter; } if (peer_) { video_track.RemoveFromPeerConnection(*peer_); } local_video_tracks_.erase(it); - return webrtc::RTCError::OK(); + return Result::kSuccess; } void PeerConnectionImpl::RemoveLocalVideoTracksFromSource( @@ -665,60 +865,119 @@ void PeerConnectionImpl::RemoveLocalVideoTracksFromSource( } } -void PeerConnectionImpl::SetLocalAudioTrackEnabled(bool enabled) noexcept { - if (local_audio_track_) { - local_audio_track_->set_enabled(enabled); +ErrorOr> PeerConnectionImpl::AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept { + if (IsClosed()) { + return Error(Result::kInvalidOperation, "The peer connection is closed."); } -} -bool PeerConnectionImpl::IsLocalAudioTrackEnabled() const noexcept { - if (local_audio_track_) { - return local_audio_track_->enabled(); + std::string name; + if (!IsStringNullOrEmpty(config.name)) { + name = config.name; + } else { + name = rtc::CreateRandomUuid(); + } + if (!SdpIsValidToken(name)) { + rtc::StringBuilder str("Invalid audio transceiver name: "); + str << name; + return Error(Result::kInvalidParameter, str.Release()); + } + + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + // Plan B doesn't have transceivers; just create a wrapper. + mline_index = (int)transceivers_.size(); // append + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), + config.transceiver_interop_handle); + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the low-level implementation object + webrtc::RtpTransceiverInit init{}; + init.direction = Transceiver::ToRtp(config.desired_direction); + init.stream_ids = Transceiver::DecodeStreamIDs(config.stream_ids); + if (!name.empty()) { + // Prepend transceiver name as first stream ID for track pairing + init.stream_ids.insert(init.stream_ids.begin(), name); + } + webrtc::RTCErrorOr> + ret = + peer_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_AUDIO, init); + if (!ret.ok()) { + return ErrorFromRTCError(ret.MoveError()); + } + rtc::scoped_refptr impl(ret.MoveValue()); + + // Find the mline index from the position inside the transceiver list + { + auto transceivers = peer_->GetTransceivers(); + auto it_tr = + std::find_if(transceivers.begin(), transceivers.end(), + [&impl](auto const& tr) { return (tr == impl); }); + RTC_DCHECK(it_tr != transceivers.end()); + mline_index = (int)std::distance(transceivers.begin(), it_tr); + } + + // Create the transceiver wrapper + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + config.transceiver_interop_handle); + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic."); } - return false; + RTC_DCHECK(wrapper); + InsertTransceiverAtMlineIndex(mline_index, wrapper); + return wrapper; } -bool PeerConnectionImpl::AddLocalAudioTrack( - rtc::scoped_refptr audio_track) noexcept { - if (local_audio_track_) { - return false; +ErrorOr> PeerConnectionImpl::AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept { + if (IsClosed()) { + return Microsoft::MixedReality::WebRTC::Error( + Result::kInvalidOperation, "The peer connection is closed."); } - if (local_audio_sender_) { - // Reuse the existing sender. - if (local_audio_sender_->SetTrack(audio_track.get())) { - if (auto* sink = local_audio_observer_.get()) { - // FIXME - Current implementation of AddSink() for the local audio - // capture device is no-op. So this callback is never fired. - audio_track->AddSink(sink); - } - local_audio_track_ = std::move(audio_track); - return true; - } - } else if (peer_) { - // Create a new sender. - auto result = peer_->AddTrack(audio_track, {kAudioVideoStreamId}); - if (result.ok()) { - if (auto* sink = local_audio_observer_.get()) { - // FIXME - Current implementation of AddSink() for the local audio - // capture device is no-op. So this callback is never fired. - audio_track->AddSink(sink); - } - local_audio_sender_ = result.value(); - local_audio_track_ = std::move(audio_track); - return true; - } + auto result = peer_->AddTrack(audio_track, {kAudioVideoStreamId}); + if (!result.ok()) { + return ErrorFromRTCError(result.MoveError()); + } + rtc::scoped_refptr sender = result.MoveValue(); + ErrorOr> ret = + GetOrCreateAudioTransceiverForSender(sender, transceiver_interop_handle); + if (!ret.ok()) { + peer_->RemoveTrack(sender); + return ret.MoveError(); + } + RefPtr transceiver = ret.MoveValue(); + RefPtr track = new LocalAudioTrack( + global_factory_, *this, std::move(transceiver), std::move(audio_track), + std::move(sender), track_interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + local_audio_tracks_.push_back(track); } - return false; + return track; } -void PeerConnectionImpl::RemoveLocalAudioTrack() noexcept { - if (!local_audio_track_) - return; - if (auto* sink = local_audio_observer_.get()) { - local_audio_track_->RemoveSink(sink); +Result PeerConnectionImpl::RemoveLocalAudioTrack( + LocalAudioTrack& audio_track) noexcept { + rtc::CritScope lock(&tracks_mutex_); + auto it = std::find_if(local_audio_tracks_.begin(), local_audio_tracks_.end(), + [&audio_track](const RefPtr& track) { + return track.get() == &audio_track; + }); + if (it == local_audio_tracks_.end()) { + return Result::kInvalidParameter; + } + if (peer_) { + audio_track.RemoveFromPeerConnection(*peer_); } - local_audio_sender_->SetTrack(nullptr); - local_audio_track_ = nullptr; + local_audio_tracks_.erase(it); + return Result::kSuccess; } ErrorOr> PeerConnectionImpl::AddDataChannel( @@ -886,6 +1145,19 @@ void PeerConnectionImpl::OnDataChannelAdded( } } +void PeerConnectionImpl::OnStreamChanged( + rtc::scoped_refptr stream) noexcept { + webrtc::AudioTrackVector audio_tracks = stream->GetAudioTracks(); + webrtc::VideoTrackVector video_tracks = stream->GetVideoTracks(); + RTC_LOG(LS_INFO) << "Media stream #" << stream->id() + << " changed: " << audio_tracks.size() + << " audio tracks and " << video_tracks.size() + << " video tracks."; + // for (auto&& audio_track : remote_audio_tracks_) { + // if (audio_track == audio_tracks[0]) + //} +} + bool PeerConnectionImpl::AddIceCandidate(const char* sdp_mid, const int sdp_mline_index, const char* candidate) noexcept { @@ -908,18 +1180,13 @@ bool PeerConnectionImpl::CreateOffer() noexcept { if (!peer_) { return false; } - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; - /*if (mandatory_receive_)*/ { //< TODO - This is legacy, should use - // transceivers - options.offer_to_receive_audio = true; - options.offer_to_receive_video = true; - } { auto lock = std::scoped_lock{data_channel_mutex_}; if (data_channels_.empty()) { sctp_negotiated_ = false; } } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options{}; auto observer = new rtc::RefCountedObject(this); // 0 ref peer_->CreateOffer(observer, options); @@ -931,12 +1198,7 @@ bool PeerConnectionImpl::CreateAnswer() noexcept { if (!peer_) { return false; } - webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options; - /*if (mandatory_receive_)*/ { //< TODO - This is legacy, should use - // transceivers - options.offer_to_receive_audio = true; - options.offer_to_receive_video = true; - } + webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options{}; auto observer = new rtc::RefCountedObject(this); // 0 ref peer_->CreateAnswer(observer, options); @@ -952,33 +1214,57 @@ void PeerConnectionImpl::Close() noexcept { // Close the connection peer_->Close(); - // Remove local tracks { rtc::CritScope lock(&tracks_mutex_); + + // Remove local tracks while (!local_video_tracks_.empty()) { RefPtr& ptr = local_video_tracks_.back(); RemoveLocalVideoTrack(*ptr); } - } + while (!local_audio_tracks_.empty()) { + RefPtr& ptr = local_audio_tracks_.back(); + RemoveLocalAudioTrack(*ptr); + } - // Ensure that observers (sinks) are removed, otherwise the media pipelines - // will continue to try to feed them with data after they're destroyed - // RemoveLocalVideoTrack(); TODO - do we need to keep a list of local tracks - // and do something here? - RemoveLocalAudioTrack(); - local_audio_sender_ = nullptr; - for (auto stream : remote_streams_) { - if (auto* sink = remote_video_observer_.get()) { - for (auto&& video_track : stream->GetVideoTracks()) { - video_track->RemoveSink(sink); + // Force-remove remote tracks. It doesn't look like the TrackRemoved + // callback is called when Close() is used, so force it here. + auto rat = std::move(remote_audio_tracks_); + auto rvt = std::move(remote_video_tracks_); + auto cb_lock = std::scoped_lock{media_track_callback_mutex_}; + auto audio_cb = audio_track_removed_callback_; + for (auto&& track : rat) { + track->OnTrackRemoved(*this); + if (auto interop_handle = track->GetInteropHandle()) { + if (audio_cb) { + auto transceiver = track->GetTransceiver(); + auto transceiver_interop_handle = transceiver->GetInteropHandle(); + audio_cb(interop_handle, track.get(), transceiver_interop_handle, + transceiver); + } } } - if (auto* sink = remote_audio_observer_.get()) { - for (auto&& audio_track : stream->GetAudioTracks()) { - audio_track->RemoveSink(sink); + auto video_cb = video_track_removed_callback_; + for (auto&& track : rvt) { + track->OnTrackRemoved(*this); + if (auto interop_handle = track->GetInteropHandle()) { + if (video_cb) { + auto transceiver = track->GetTransceiver(); + auto transceiver_interop_handle = transceiver->GetInteropHandle(); + video_cb(interop_handle, track.get(), transceiver_interop_handle, + transceiver); + } } } + + // Clear transceivers + //< TODO - This is done inside the lock, but the lock is released before + // peer_ is cleared, so before the connection is actually closed, which + // doesn't prevent add(Audio|Video)Transceiver from being called again in + // parallel... + transceivers_.clear(); } + remote_streams_.clear(); RemoveAllDataChannels(); @@ -1020,6 +1306,22 @@ bool PeerConnectionImpl::SetRemoteDescriptionAsync( return false; rtc::scoped_refptr observer = new rtc::RefCountedObject([this, callback] { + // Inspect transceiver directions, check for changes to update the + // interop layer with the actually negotiated direction. + std::vector> + changed_transceivers; + int mline_index = 0; // native transceivers are in mline_index order + for (auto&& tr : peer_->GetTransceivers()) { + // If transceiver is created from the result of applying a remote + // description, then the transceiver name is extracted from the + // receiver, in an attempt to pair with the remote peer's track. + std::string name = ExtractTransceiverNameFromReceiver(tr->receiver()); + ErrorOr> err = + GetOrCreateTransceiver(mline_index, tr, std::move(name)); + RTC_DCHECK(err.ok()); + err.value()->OnSessionDescUpdated(/*remote=*/true); + ++mline_index; + } // Fire completed callback to signal remote description was applied. callback(); }); @@ -1058,7 +1360,9 @@ void PeerConnectionImpl::OnAddStream( RTC_LOG(LS_INFO) << "Added stream #" << stream->id() << " with " << stream->GetAudioTracks().size() << " audio tracks and " << stream->GetVideoTracks().size() << " video tracks."; - remote_streams_.push_back(stream); + auto observer = std::make_unique(*this, stream); + stream->RegisterObserver(observer.get()); + remote_streams_.emplace(std::move(observer), stream); } void PeerConnectionImpl::OnRemoveStream( @@ -1066,10 +1370,17 @@ void PeerConnectionImpl::OnRemoveStream( RTC_LOG(LS_INFO) << "Removed stream #" << stream->id() << " with " << stream->GetAudioTracks().size() << " audio tracks and " << stream->GetVideoTracks().size() << " video tracks."; - auto it = std::find(remote_streams_.begin(), remote_streams_.end(), stream); + auto it = std::find_if( + remote_streams_.begin(), remote_streams_.end(), + [&stream]( + std::pair, + rtc::scoped_refptr>& pair) { + return pair.second == stream; + }); if (it == remote_streams_.end()) { return; } + stream->UnregisterObserver(it->first.get()); remote_streams_.erase(it); } @@ -1188,47 +1499,145 @@ void PeerConnectionImpl::OnAddTrack( rtc::scoped_refptr receiver, const std::vector>& /*streams*/) noexcept { - RTC_LOG(LS_INFO) << "Added track #" << receiver->id() << " of type " + RTC_LOG(LS_INFO) << "Added receiver #" << receiver->id() << " of type " << (int)receiver->media_type(); for (auto&& stream : receiver->streams()) { - RTC_LOG(LS_INFO) << "+ Track #" << receiver->id() << " with stream #" - << stream->id(); + RTC_LOG(LS_INFO) << "+ Track #" << receiver->track()->id() + << " with stream #" << stream->id(); } - // Register the remote observer + // Create the remote track wrapper rtc::scoped_refptr track = receiver->track(); - TrackKind trackKind = TrackKind::kUnknownTrack; - const std::string& trackKindStr = track->kind(); - if (trackKindStr == webrtc::MediaStreamTrackInterface::kAudioKind) { - trackKind = TrackKind::kAudioTrack; - if (auto* sink = remote_audio_observer_.get()) { - auto audio_track = static_cast(track.get()); - audio_track->AddSink(sink); + const std::string& track_name = track->id(); + const std::string& track_kind_str = track->kind(); + if (track_kind_str == webrtc::MediaStreamTrackInterface::kAudioKind) { + rtc::scoped_refptr audio_track( + static_cast(track.release())); + + // Create an interop wrapper for the new native object if needed + mrsRemoteAudioTrackInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.remote_audio_track_create_object) { + mrsRemoteAudioTrackConfig config; + config.track_name = track_name.c_str(); + interop_handle = (*create_cb)(interop_handle_, config); } - } else if (trackKindStr == webrtc::MediaStreamTrackInterface::kVideoKind) { - trackKind = TrackKind::kVideoTrack; - if (auto* sink = remote_video_observer_.get()) { - rtc::VideoSinkWants sink_settings{}; - sink_settings.rotation_applied = - true; // no exposed API for caller to handle rotation - auto video_track = static_cast(track.get()); - video_track->AddOrUpdateSink(sink, sink_settings); + + // Get or create the transceiver wrapper based on the RTP receiver. Because + // this callback is fired before the one at the end of the remote + // description being applied, the transceiver wrappers for the newly added + // RTP transceivers have not been created yet, so create them here. + auto ret = GetOrCreateAudioTransceiverForNewRemoteTrack(receiver); + if (!ret.ok()) { + return; } - } else { - return; - } + RefPtr transceiver = ret.MoveValue(); - // Invoke the TrackAdded callback - { - auto lock = std::scoped_lock{track_added_callback_mutex_}; - auto cb = track_added_callback_; - if (cb) { - cb(trackKind); + // The transceiver wrapper might have been created, in which case we need to + // inform its interop wrapper of its handle. + mrsAudioTransceiverInteropHandle transceiver_interop_handle = + transceiver->GetInteropHandle(); + + // Create the native object + RefPtr remote_audio_track = new RemoteAudioTrack( + global_factory_, *this, transceiver, std::move(audio_track), + std::move(receiver), interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + remote_audio_tracks_.emplace_back(remote_audio_track); + } + + // Invoke the AudioTrackAdded callback, which will set the native handle on + // the interop wrapper (if created above) + { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = audio_track_added_callback_; + if (cb) { + AudioTransceiverHandle tranceiver_handle = transceiver.release(); + RemoteAudioTrackHandle audio_handle = remote_audio_track.release(); + cb(interop_handle, audio_handle, transceiver_interop_handle, + tranceiver_handle); + } + } + } else if (track_kind_str == webrtc::MediaStreamTrackInterface::kVideoKind) { + rtc::scoped_refptr video_track( + static_cast(track.release())); + + // Create an interop wrapper for the new native object if needed + mrsRemoteVideoTrackInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.remote_video_track_create_object) { + mrsRemoteVideoTrackConfig config; + config.track_name = track_name.c_str(); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Get or create the transceiver wrapper based on the RTP receiver. Because + // this callback is fired before the one at the end of the remote + // description being applied, the transceiver wrappers for the newly added + // RTP transceivers have not been created yet, so create them here. + auto ret = GetOrCreateVideoTransceiverForRemoteNewTrack(receiver); + if (!ret.ok()) { + return; + } + RefPtr transceiver = ret.MoveValue(); + + // The transceiver wrapper might have been created, in which case we need to + // inform its interop wrapper of its handle. + mrsVideoTransceiverInteropHandle transceiver_interop_handle = + transceiver->GetInteropHandle(); + + // Create the native object + RefPtr remote_video_track = new RemoteVideoTrack( + global_factory_, *this, transceiver, std::move(video_track), + std::move(receiver), interop_handle); + { + rtc::CritScope lock(&tracks_mutex_); + remote_video_tracks_.emplace_back(remote_video_track); + } + + // Invoke the VideoTrackAdded callback, which will set the native handle on + // the interop wrapper (if created above) + { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = video_track_added_callback_; + if (cb) { + VideoTransceiverHandle tranceiver_handle = transceiver.release(); + RemoteVideoTrackHandle video_handle = remote_video_track.release(); + cb(interop_handle, video_handle, transceiver_interop_handle, + tranceiver_handle); + } } } } +void PeerConnectionImpl::OnTrack( + rtc::scoped_refptr transceiver) noexcept { + RTC_LOG(LS_INFO) << "Added transceiver mid=#" << transceiver->mid().value() + << " of type " << (int)transceiver->media_type() + << " with desired direction " + << (int)transceiver->direction(); + auto receiver = transceiver->receiver(); + if (auto track = receiver->track()) { + RTC_LOG(LS_INFO) << "Recv with track #" << track->id() + << " enabled=" << track->enabled(); + } else { + RTC_LOG(LS_INFO) << "Recv with NULL track"; + } + for (auto&& id : receiver->stream_ids()) { + RTC_LOG(LS_INFO) << "+ Stream #" << id; + } + auto sender = transceiver->sender(); + if (auto track = sender->track()) { + RTC_LOG(LS_INFO) << "Send #" << track->id() + << " enabled=" << track->enabled(); + } else { + RTC_LOG(LS_INFO) << "Send with NULL track"; + } + for (auto&& id : sender->stream_ids()) { + RTC_LOG(LS_INFO) << "+ Stream #" << id; + } +} + void PeerConnectionImpl::OnRemoveTrack( rtc::scoped_refptr receiver) noexcept { RTC_LOG(LS_INFO) << "Removed track #" << receiver->id() << " of type " @@ -1241,31 +1650,59 @@ void PeerConnectionImpl::OnRemoveTrack( // Unregister the remote observer rtc::scoped_refptr track = receiver->track(); - TrackKind trackKind = TrackKind::kUnknownTrack; - const std::string& trackKindStr = track->kind(); - if (trackKindStr == webrtc::MediaStreamTrackInterface::kAudioKind) { - trackKind = TrackKind::kAudioTrack; - if (auto* sink = remote_audio_observer_.get()) { - auto audio_track = static_cast(track.get()); - audio_track->RemoveSink(sink); + const std::string& track_kind_str = track->kind(); + if (track_kind_str == webrtc::MediaStreamTrackInterface::kAudioKind) { + rtc::CritScope tracks_lock(&tracks_mutex_); + auto it = + std::find_if(remote_audio_tracks_.begin(), remote_audio_tracks_.end(), + [&receiver](const RefPtr& remote_track) { + return (remote_track->receiver() == receiver); + }); + if (it == remote_audio_tracks_.end()) { + return; } - } else if (trackKindStr == webrtc::MediaStreamTrackInterface::kVideoKind) { - trackKind = TrackKind::kVideoTrack; - if (auto* sink = remote_video_observer_.get()) { - auto video_track = static_cast(track.get()); - video_track->RemoveSink(sink); + RefPtr audio_track = std::move(*it); + RefPtr audio_transceiver = audio_track->GetTransceiver(); + remote_audio_tracks_.erase(it); + audio_track->OnTrackRemoved(*this); + + // Invoke the TrackRemoved callback + if (auto interop_handle = audio_track->GetInteropHandle()) { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = audio_track_removed_callback_; + if (cb) { + auto transceiver_interop_handle = audio_transceiver->GetInteropHandle(); + cb(interop_handle, audio_track.get(), transceiver_interop_handle, + audio_transceiver.get()); + } } - } else { - return; - } - - // Invoke the TrackRemoved callback - { - auto lock = std::scoped_lock{track_removed_callback_mutex_}; - auto cb = track_removed_callback_; - if (cb) { - cb(trackKind); + // |audio_track| goes out of scope and destroys the C++ instance + } else if (track_kind_str == webrtc::MediaStreamTrackInterface::kVideoKind) { + rtc::CritScope tracks_lock(&tracks_mutex_); + auto it = + std::find_if(remote_video_tracks_.begin(), remote_video_tracks_.end(), + [&receiver](const RefPtr& remote_track) { + return (remote_track->receiver() == receiver); + }); + if (it == remote_video_tracks_.end()) { + return; + } + RefPtr video_track = std::move(*it); + RefPtr video_transceiver = video_track->GetTransceiver(); + remote_video_tracks_.erase(it); + video_track->OnTrackRemoved(*this); + + // Invoke the TrackRemoved callback + if (auto interop_handle = video_track->GetInteropHandle()) { + auto lock = std::scoped_lock{media_track_callback_mutex_}; + auto cb = video_track_removed_callback_; + if (cb) { + auto transceiver_interop_handle = video_transceiver->GetInteropHandle(); + cb(interop_handle, video_track.get(), transceiver_interop_handle, + video_transceiver.get()); + } } + // |video_track| goes out of scope and destroys the C++ instance } } @@ -1274,28 +1711,544 @@ void PeerConnectionImpl::OnLocalDescCreated( if (!peer_) { return; } - auto lock = std::scoped_lock{local_sdp_ready_to_send_callback_mutex_}; - auto cb = local_sdp_ready_to_send_callback_; - rtc::scoped_refptr observer; - if (cb) { - std::string type{SdpTypeToString(desc->GetType())}; - ensureNullTerminatedCString(type); - std::string sdp; - desc->ToString(&sdp); - ensureNullTerminatedCString(sdp); - observer = new rtc::RefCountedObject( - [cb, type = std::move(type), sdp = std::move(sdp)] { - cb(type.c_str(), sdp.c_str()); - }); - } else { - observer = new rtc::RefCountedObject(); - } + rtc::scoped_refptr observer = + new rtc::RefCountedObject([this] { + // Inspect transceiver directions, check for changes to update the + // interop layer with the actually negotiated direction. + std::vector> + changed_transceivers; + int mline_index = 0; // native transceivers are in mline_index order + for (auto&& tr : peer_->GetTransceivers()) { + // If transceiver is created from the result of applying a local + // description, then the transceiver name is extracted from the + // sender, as the name should have been set by the user. + std::string name = ExtractTransceiverNameFromSender(tr->sender()); + ErrorOr> err = + GetOrCreateTransceiver(mline_index, tr, std::move(name)); + RTC_DCHECK(err.ok()); + err.value()->OnSessionDescUpdated(/*remote=*/false); + ++mline_index; + } + + // Fire interop callback, if any + { + auto lock = std::scoped_lock{local_sdp_ready_to_send_callback_mutex_}; + if (auto cb = local_sdp_ready_to_send_callback_) { + auto desc = peer_->local_description(); + std::string type{SdpTypeToString(desc->GetType())}; + std::string sdp; + desc->ToString(&sdp); + cb(type.c_str(), sdp.c_str()); + } + } + }); // SetLocalDescription will invoke observer.OnSuccess() once done, which // will in turn invoke the |local_sdp_ready_to_send_callback_| registered if // any, or do nothing otherwise. The observer is a mandatory parameter. peer_->SetLocalDescription(observer, desc); } +void PeerConnectionImpl::OnLocalTrackAddedToAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kAudio)); + }) != transceivers_.end()); + RTC_DCHECK(std::find_if(local_audio_tracks_.begin(), + local_audio_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }) == local_audio_tracks_.end()); + local_audio_tracks_.push_back(&track); +} + +void PeerConnectionImpl::OnLocalTrackRemovedFromAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kAudio)); + }) != transceivers_.end()); + auto it = std::find_if(local_audio_tracks_.begin(), local_audio_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }); + RTC_DCHECK(it != local_audio_tracks_.end()); + local_audio_tracks_.erase(it); +} + +void PeerConnectionImpl::OnLocalTrackAddedToVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kVideo)); + }) != transceivers_.end()); + RTC_DCHECK(std::find_if(local_video_tracks_.begin(), + local_video_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }) == local_video_tracks_.end()); + local_video_tracks_.push_back(&track); +} + +void PeerConnectionImpl::OnLocalTrackRemovedFromVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) { + rtc::CritScope lock(&tracks_mutex_); + RTC_DCHECK(std::find_if(transceivers_.begin(), transceivers_.end(), + [&transceiver](const RefPtr& tr) { + return ((tr.get() == &transceiver) && + (tr->GetMediaKind() == MediaKind::kVideo)); + }) != transceivers_.end()); + auto it = std::find_if(local_video_tracks_.begin(), local_video_tracks_.end(), + [&track](const RefPtr& tr) { + return (tr.get() == &track); + }); + RTC_DCHECK(it != local_video_tracks_.end()); + local_video_tracks_.erase(it); +} + +Error PeerConnectionImpl::InsertTransceiverAtMlineIndex( + int mline_index, + RefPtr transceiver) { + RTC_CHECK(mline_index >= 0); + rtc::CritScope lock(&tracks_mutex_); + if (mline_index >= transceivers_.size()) { + // Insert empty entries for now; they should be filled when processing + // other added remote tracks or when finishing the transceiver status + // update. + while (mline_index >= transceivers_.size() + 1) { + transceivers_.push_back(nullptr); + } + transceivers_.push_back(transceiver); + } else { + if (transceivers_[mline_index]) { + RTC_LOG(LS_ERROR) << "Trying to insert transceiver (name=" + << transceiver->GetName().c_str() + << ") at mline index #" << mline_index + << ", but another transceiver (name=" + << transceivers_[mline_index]->GetName().c_str() + << ") already exists with the same index."; + return Error(Result::kUnknownError, + "Duplicate transceiver for mline index"); + } + transceivers_[mline_index] = transceiver; + } + return Error(Result::kSuccess); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateAudioTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsAudioTransceiverInteropHandle transceiver_interop_handle) { + RTC_DCHECK(sender->media_type() == cricket::MediaType::MEDIA_TYPE_AUDIO); + + // Find existing transceiver for sender + auto it = std::find_if(transceivers_.begin(), transceivers_.end(), + [sender](const RefPtr& tr) { + return (tr && (tr->impl()->sender() == sender)); + }); + if (it != transceivers_.end()) { + RTC_DCHECK(MediaKind::kAudio == it->get()->GetMediaKind()); + RefPtr audio_transceiver = + static_cast(it->get()); + RTC_DCHECK_EQ(transceiver_interop_handle, + audio_transceiver->GetInteropHandle()); + return audio_transceiver; + } + + std::string name = ExtractTransceiverNameFromSender(sender); + + // Create new transceiver wrapper for a newly created local audio track. + // This is called from AddLocalAudioTrack() only, after calling the lower + // level AddTrack() which may reuse an existing transceiver, so trust the + // implementation for the mline index. + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Find transceiver implementation. It doesn't seem like there is a direct + // back-link, so iterate over all the peer connection transceivers. + auto transceivers = peer_->GetTransceivers(); + auto it_tr = std::find_if( + transceivers.begin(), transceivers.end(), + [sender](auto const& tr) { return (tr->sender() == sender); }); + if (it_tr == transceivers.end()) { + return Error(Result::kInvalidOperation, + "Cannot match RTP sender with RTP transceiver."); + } + rtc::scoped_refptr impl = *it_tr; + mline_index = (int)std::distance(transceivers.begin(), it_tr); + + // Create the transceiver wrapper + wrapper = new AudioTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + transceiver_interop_handle); + + // Note: at this point the native wrapper knows about the interop wrapper, + // but not the opposite. Normally we'd fire another "created-callback" + // with the native wrapper handle to sync the interop wrapper, but here it + // is being created as part of a local track creation, so we bundle that + // with the "track-created" event. + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (wrapper) { + auto err = InsertTransceiverAtMlineIndex(mline_index, wrapper); + if (!err.ok()) { + return err; + } + return wrapper; + } + return Error(Result::kUnknownError, + "Failed to create a new transceiver for local audio track."); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateVideoTransceiverForSender( + webrtc::RtpSenderInterface* sender, + mrsVideoTransceiverInteropHandle transceiver_interop_handle) { + RTC_DCHECK(sender->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO); + + // Find existing transceiver for sender + auto it = std::find_if(transceivers_.begin(), transceivers_.end(), + [sender](const RefPtr& tr) { + return (tr && (tr->impl()->sender() == sender)); + }); + if (it != transceivers_.end()) { + RTC_DCHECK(MediaKind::kVideo == it->get()->GetMediaKind()); + RefPtr video_transceiver = + static_cast(it->get()); + RTC_DCHECK_EQ(transceiver_interop_handle, + video_transceiver->GetInteropHandle()); + return video_transceiver; + } + + std::string name = ExtractTransceiverNameFromSender(sender); + + // Create new transceiver for video track + RefPtr wrapper; + int mline_index = -1; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Find transceiver implementation. It doesn't seem like there is a direct + // back-link, so iterate over all the peer connection transceivers. + auto transceivers = peer_->GetTransceivers(); + auto it_tr = std::find_if( + transceivers.begin(), transceivers.end(), + [sender](auto const& tr) { return (tr->sender() == sender); }); + if (it_tr == transceivers.end()) { + return Error(Result::kInvalidOperation, + "Cannot match RTP sender with RTP transceiver."); + } + rtc::scoped_refptr impl = *it_tr; + mline_index = (int)std::distance(transceivers.begin(), it_tr); + + // Create the transceiver wrapper + wrapper = new VideoTransceiver(global_factory_, *this, mline_index, + std::move(name), std::move(impl), + transceiver_interop_handle); + + // Note: at this point the native wrapper knows about the interop wrapper, + // but not the opposite. Normally we'd fire another "created-callback" + // with the native wrapper handle to sync the interop wrapper, but here it + // is being created as part of a local track creation, so we bundle that + // with the "track-created" event. + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (wrapper) { + auto err = InsertTransceiverAtMlineIndex(mline_index, wrapper); + if (!err.ok()) { + return err; + } + return wrapper; + } + return Error(Result::kUnknownError, + "Failed to create a new transceiver for local video track."); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateAudioTransceiverForNewRemoteTrack( + webrtc::RtpReceiverInterface* receiver) { + RTC_DCHECK(receiver->media_type() == cricket::MediaType::MEDIA_TYPE_AUDIO); + + // Try to find an existing audio transceiver wrapper for the given RTP + // receiver of the remote track. + { + auto it_tr = std::find_if(transceivers_.begin(), transceivers_.end(), + [receiver](const RefPtr& tr) { + return (tr->impl()->receiver() == receiver); + }); + if (it_tr != transceivers_.end()) { + RTC_DCHECK(MediaKind::kAudio == (*it_tr)->GetMediaKind()); + RefPtr audio_transceiver = + static_cast(it_tr->get()); + return audio_transceiver; + } + } + + // The new remote track should already have a low-level implementation RTP + // transceiver from applying the remote description. But the wrapper for it + // was not created yet. Find the RTP transceiver of the RTP receiver, bearing + // in mind its mline index is not necessarily contiguous in the wrapper array. + auto transceivers = peer_->GetTransceivers(); + auto it_impl = std::find_if( + transceivers.begin(), transceivers.end(), + [receiver](auto&& tr) { return tr->receiver() == receiver; }); + if (it_impl == transceivers.end()) { + return Error( + Result::kNotFound, + "Failed to match RTP receiver with an existing RTP transceiver."); + } + rtc::scoped_refptr impl = *it_impl; + const int mline_index = (int)std::distance(transceivers.begin(), it_impl); + + std::string name = ExtractTransceiverNameFromReceiver(receiver); + + // Create a new audio transceiver wrapper for it + return CreateAudioTransceiver(mline_index, std::move(name), std::move(impl)); +} + +ErrorOr> +PeerConnectionImpl::GetOrCreateVideoTransceiverForRemoteNewTrack( + webrtc::RtpReceiverInterface* receiver) { + RTC_DCHECK(receiver->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO); + + // Try to find an existing video transceiver wrapper for the given RTP + // receiver of the remote track. + { + auto it_tr = std::find_if(transceivers_.begin(), transceivers_.end(), + [receiver](const RefPtr& tr) { + return (tr->impl()->receiver() == receiver); + }); + if (it_tr != transceivers_.end()) { + RTC_DCHECK(MediaKind::kVideo == (*it_tr)->GetMediaKind()); + RefPtr video_transceiver = + static_cast(it_tr->get()); + return video_transceiver; + } + } + + // The new remote track should already have a low-level implementation RTP + // transceiver from applying the remote description. But the wrapper for it + // was not created yet. Find the RTP transceiver of the RTP receiver, bearing + // in mind its mline index is not necessarily contiguous in the wrapper array. + auto transceivers = peer_->GetTransceivers(); + auto it_impl = std::find_if( + transceivers.begin(), transceivers.end(), + [receiver](auto&& tr) { return tr->receiver() == receiver; }); + if (it_impl == transceivers.end()) { + return Error( + Result::kNotFound, + "Failed to match RTP receiver with an existing RTP transceiver."); + } + rtc::scoped_refptr impl = *it_impl; + const int mline_index = (int)std::distance(transceivers.begin(), it_impl); + + std::string name = ExtractTransceiverNameFromReceiver(receiver); + + // Create a new video transceiver wrapper for it + return CreateVideoTransceiver(mline_index, std::move(name), std::move(impl)); +} + +ErrorOr> PeerConnectionImpl::GetOrCreateTransceiver( + int mline_index, + webrtc::RtpTransceiverInterface* rtp_transceiver, + std::string name) { + switch (rtp_transceiver->media_type()) { + case cricket::MediaType::MEDIA_TYPE_AUDIO: { + // Find an existing transceiver wrapper which would have been created just + // a moment ago by the remote track added callback. + rtc::CritScope lock(&tracks_mutex_); + for (auto&& tr : transceivers_) { + if (tr && (tr->impl() == rtp_transceiver)) { + RTC_DCHECK(MediaKind::kAudio == tr->GetMediaKind()); + return static_cast>(tr); + } + } + // Not found - create a new one + return CreateAudioTransceiver(mline_index, std::move(name), + rtp_transceiver); + }; + + case cricket::MediaType::MEDIA_TYPE_VIDEO: { + // Find an existing transceiver wrapper which would have been created just + // a moment ago by the remote track added callback. + rtc::CritScope lock(&tracks_mutex_); + for (auto&& tr : transceivers_) { + if (tr && (tr->impl() == rtp_transceiver)) { + RTC_DCHECK(MediaKind::kVideo == tr->GetMediaKind()); + return static_cast>(tr); + } + } + // Not found - create a new one + return CreateVideoTransceiver(mline_index, std::move(name), + rtp_transceiver); + }; + + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } +} + +ErrorOr> PeerConnectionImpl::CreateAudioTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver) { + // Create an interop wrapper for the new native object if needed + mrsAudioTransceiverInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.audio_transceiver_create_object) { + mrsAudioTransceiverConfig config{}; + config.name = name.c_str(); + config.mline_index = mline_index; + config.initial_desired_direction = + Transceiver::FromRtp(rtp_transceiver->direction()); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Create new transceiver wrapper + RefPtr transceiver; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the transceiver wrapper + transceiver = new AudioTransceiver( + global_factory_, *this, mline_index, std::move(name), + std::move(rtp_transceiver), interop_handle); + + // Synchronize the interop wrapper with the current object. + if (auto cb = interop_callbacks_.audio_transceiver_finish_create) { + transceiver->AddRef(); + cb(interop_handle, transceiver.get()); + } + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (transceiver) { + auto err = InsertTransceiverAtMlineIndex(mline_index, transceiver); + if (!err.ok()) { + return err; + } + } + return transceiver; +} + +ErrorOr> PeerConnectionImpl::CreateVideoTransceiver( + int mline_index, + std::string name, + rtc::scoped_refptr rtp_transceiver) { + // Create an interop wrapper for the new native object if needed + mrsVideoTransceiverInteropHandle interop_handle{}; + if (auto create_cb = interop_callbacks_.video_transceiver_create_object) { + mrsVideoTransceiverConfig config{}; + config.name = name.c_str(); + config.mline_index = mline_index; + config.initial_desired_direction = + Transceiver::FromRtp(rtp_transceiver->direction()); + interop_handle = (*create_cb)(interop_handle_, config); + } + + // Create new transceiver wrapper + RefPtr transceiver; + switch (peer_->GetConfiguration().sdp_semantics) { + case webrtc::SdpSemantics::kPlanB: { + assert(false); //< TODO... + } break; + case webrtc::SdpSemantics::kUnifiedPlan: { + // Create the transceiver wrapper + transceiver = new VideoTransceiver( + global_factory_, *this, mline_index, std::move(name), + std::move(rtp_transceiver), interop_handle); + + // Synchronize the interop wrapper with the current object. + if (auto cb = interop_callbacks_.video_transceiver_finish_create) { + transceiver->AddRef(); + cb(interop_handle, transceiver.get()); + } + } break; + default: + return Error(Result::kUnknownError, "Unknown SDP semantic"); + } + if (transceiver) { + auto err = InsertTransceiverAtMlineIndex(mline_index, transceiver); + if (!err.ok()) { + return err; + } + } + return transceiver; +} + +std::string PeerConnectionImpl::ExtractTransceiverNameFromSender( + webrtc::RtpSenderInterface* sender) { + // Find the pairing name as the first stream ID. + // See |LocalAudioTrack::GetName()|, |RemoteAudioTrack::GetName()|, + // |LocalVideoTrack::GetName()|, |RemoteVideoTrack::GetName()|. + auto ids = sender->stream_ids(); + if (!ids.empty()) { + return ids[0]; + } + // Fallback on track's ID, even though it's not pairable in Unified Plan (and + // technically neither in Plan B, although this works in practice). + if (rtc::scoped_refptr track = + sender->track()) { + return track->id(); + } + return {}; +} + +std::string PeerConnectionImpl::ExtractTransceiverNameFromReceiver( + webrtc::RtpReceiverInterface* receiver) { + // Find the pairing name as the first stream ID. + // See |LocalAudioTrack::GetName()|, |RemoteAudioTrack::GetName()|, + // |LocalVideoTrack::GetName()|, |RemoteVideoTrack::GetName()|. + { + // BUG + // webrtc::RtpReceiverInterface::stream_ids() is not proxied correctly, does + // not resolve to its implementation and instead always returns an empty + // vector. Use ::streams() instead even if deprecated. Fixed by + // https://webrtc.googlesource.com/src/+/5b1477839d8569291b88dfe950089d0ebf34bc8f +#if 0 + auto ids = receiver->stream_ids(); + if (!ids.empty()) { + return ids[0]; + } +#else + // Use internal implementation of stream_ids() + auto streams = receiver->streams(); + if (!streams.empty()) { + return streams[0]->id(); + } +#endif + } + // Fallback on track's ID, even though it's not pairable in Unified Plan (and + // technically neither in Plan B, although this works in practice). + if (rtc::scoped_refptr track = + receiver->track()) { + return track->id(); + } + return {}; +} + webrtc::PeerConnectionInterface::IceTransportsType ICETransportTypeToNative( IceTransportType mrsValue) { using Native = webrtc::PeerConnectionInterface::IceTransportsType; @@ -1330,7 +2283,8 @@ ErrorOr> PeerConnection::create( // Ensure the factory exists RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); + rtc::scoped_refptr pc_factory = + global_factory->GetPeerConnectionFactory(); if (!pc_factory) { return Error(Result::kUnknownError); } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h index fc00aac95..a533c060f 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.h @@ -6,6 +6,8 @@ #include "audio_frame_observer.h" #include "callback.h" #include "data_channel.h" +#include "media/audio_transceiver.h" +#include "media/video_transceiver.h" #include "mrs_errors.h" #include "refptr.h" #include "tracked_object.h" @@ -14,6 +16,7 @@ namespace Microsoft::MixedReality::WebRTC { class PeerConnection; +class LocalAudioTrack; class LocalVideoTrack; class ExternalVideoTrackSource; class DataChannel; @@ -34,34 +37,36 @@ struct BitrateSettings { /// The PeerConnection class is the entry point to most of WebRTC. /// It encapsulates a single connection between a local peer and a remote peer, -/// and hosts some critical events for signaling and video rendering. +/// and hosts some critical events for signaling. /// /// The high level flow to establish a connection is as follow: /// - Create a peer connection object from a factory with /// PeerConnection::create(). -/// - Register a custom callback to the various signaling events. +/// - Register custom callbacks to the various signaling events, and dispatch +/// signaling messages back and forth using the chosen signaling solution. /// - Optionally add audio/video/data tracks. These can also be added after the /// connection is established, but see remark below. -/// - Create a peer connection offer, or wait for the remote peer to send an -/// offer, and respond with an answer. +/// - Create a peer connection offer with |CreateOffer()|, or wait for the +/// remote peer to send an offer, and respond with an answer with +/// |CreateAnswer()|. /// -/// At any point, before or after the connection is initated (CreateOffer() or -/// CreateAnswer()) or established (RegisterConnectedCallback()), some audio, -/// video, and data tracks can be added to it, with the following notable +/// At any point, before or after the connection is initated (|CreateOffer()| or +/// |CreateAnswer()|) or established (|RegisterConnectedCallback()|), some +/// audio, video, and data tracks can be added to it, with the following notable /// remarks and restrictions: /// - Data tracks use the DTLS/SCTP protocol and are encrypted; this requires a /// handshake to exchange encryption secrets. This exchange is only performed /// during the initial connection handshake if at least one data track is /// present. As a consequence, at least one data track needs to be added before -/// calling CreateOffer() or CreateAnswer() if the application ever need to use -/// data channels. Otherwise trying to add a data channel after that initial +/// calling |CreateOffer()| or |CreateAnswer()| if the application ever need to +/// use data channels. Otherwise trying to add a data channel after that initial /// handshake will always fail. /// - Adding and removing any kind of tracks after the connection has been -/// initiated result in a RenegotiationNeeded event to perform a new track -/// negotitation, which requires signaling to be working. Therefore it is +/// initiated result in a |RenegotiationNeeded| event to perform a new track +/// negotiation, which requires signaling to be working. Therefore it is /// recommended when this is known in advance to add tracks before starting to -/// establish a connection, to perform the first handshake with the correct -/// tracks offer/answer right away. +/// establish a connection, in order to perform the first handshake with the +/// correct tracks offer/answer right away. class PeerConnection : public TrackedObject { public: /// Create a new PeerConnection based on the given |config|. @@ -204,47 +209,24 @@ class PeerConnection : public TrackedObject { /// been called. virtual bool IsClosed() const noexcept = 0; - // - // Remote tracks - // - - /// Callback fired when a remote track is added to the peer connection. - using TrackAddedCallback = Callback; - - /// Register a custom TrackAddedCallback. - virtual void RegisterTrackAddedCallback( - TrackAddedCallback&& callback) noexcept = 0; - - /// Callback fired when a remote track is removed from the peer connection. - using TrackRemovedCallback = Callback; - - /// Register a custom TrackRemovedCallback. - virtual void RegisterTrackRemovedCallback( - TrackRemovedCallback&& callback) noexcept = 0; - // // Video // - /// Register a custom callback invoked when a remote video frame has been - /// received and decompressed, and is ready to be displayed locally. - virtual void RegisterRemoteVideoFrameCallback( - I420AFrameReadyCallback callback) noexcept = 0; - - /// Register a custom callback invoked when a remote video frame has been - /// received and decompressed, and is ready to be displayed locally. - virtual void RegisterRemoteVideoFrameCallback( - Argb32FrameReadyCallback callback) noexcept = 0; + virtual ErrorOr> AddVideoTransceiver( + const VideoTransceiverInitConfig& config) noexcept = 0; /// Add a video track to the peer connection. If no RTP sender/transceiver - /// exist, create a new one for that track. + /// exist, create a new one for that track. Otherwise try to reuse an existing + /// one. virtual ErrorOr> AddLocalVideoTrack( rtc::scoped_refptr video_track, - mrsLocalVideoTrackInteropHandle interop_handle) noexcept = 0; + mrsVideoTransceiverInteropHandle transceiver_interop_handle, + mrsLocalVideoTrackInteropHandle track_interop_handle) noexcept = 0; /// Remove a local video track from the peer connection. /// The underlying RTP sender/transceiver are kept alive but inactive. - virtual webrtc::RTCError RemoveLocalVideoTrack( + virtual Result RemoveLocalVideoTrack( LocalVideoTrack& video_track) noexcept = 0; /// Remove all tracks sharing the given video track source. @@ -253,6 +235,32 @@ class PeerConnection : public TrackedObject { virtual void RemoveLocalVideoTracksFromSource( ExternalVideoTrackSource& source) noexcept = 0; + /// Callback invoked when a remote video track is added to the peer + /// connection. + using VideoTrackAddedCallback = Callback; + + /// Register a custom |VideoTrackAddedCallback| invoked when a remote video + /// track is added to the peer connection. Only one callback can be registered + /// at a time. + virtual void RegisterVideoTrackAddedCallback( + VideoTrackAddedCallback&& callback) noexcept = 0; + + /// Callback invoked when a remote video track is removed from the peer + /// connection. + using VideoTrackRemovedCallback = Callback; + + /// Register a custom |VideoTrackRemovedCallback| invoked when a remote video + /// track is removed from the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterVideoTrackRemovedCallback( + VideoTrackRemovedCallback&& callback) noexcept = 0; + /// Rounding mode of video frame height for |SetFrameHeightRoundMode()|. /// This is only used on HoloLens 1 (UWP x86). enum class FrameHeightRoundMode { @@ -285,51 +293,47 @@ class PeerConnection : public TrackedObject { // Audio // - /// Register a custom callback invoked when a local audio frame is ready to be - /// output. - /// - /// FIXME - Current implementation of AddSink() for the local audio capture - /// device is no-op. So this callback is never fired. - virtual void RegisterLocalAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept = 0; - - /// Register a custom callback invoked when a remote audio frame has been - /// received and uncompressed, and is ready to be output locally. - virtual void RegisterRemoteAudioFrameCallback( - AudioFrameReadyCallback callback) noexcept = 0; - - /// Add to the peer connection an audio track backed by a local audio capture - /// device. If no RTP sender/transceiver exist, create a new one for that - /// track. - /// - /// Note: currently a single local video track is supported per peer - /// connection. - virtual bool AddLocalAudioTrack( - rtc::scoped_refptr audio_track) noexcept = 0; + virtual ErrorOr> AddAudioTransceiver( + const AudioTransceiverInitConfig& config) noexcept = 0; + + /// Add an audio track to the peer connection. If no RTP sender/transceiver + /// exist, create a new one for that track. Otherwise try to reuse an existing + /// one. + virtual ErrorOr> AddLocalAudioTrack( + rtc::scoped_refptr audio_track, + mrsAudioTransceiverInteropHandle transceiver_interop_handle, + mrsLocalAudioTrackInteropHandle track_interop_handle) noexcept = 0; - /// Remove the existing local audio track from the peer connection. + /// Remove an existing local audio track from the peer connection. /// The underlying RTP sender/transceiver are kept alive but inactive. - /// - /// Note: currently a single local audio track is supported per peer - /// connection. - virtual void RemoveLocalAudioTrack() noexcept = 0; + virtual Result RemoveLocalAudioTrack( + LocalAudioTrack& audio_track) noexcept = 0; - /// Enable or disable the local audio track. Disabled audio tracks are still - /// active but are silent, and do not consume network bandwidth. Additionally, - /// enabling/disabling the local audio track does not require an SDP exchange. - /// Therefore this is a cheaper alternative to removing and re-adding the - /// track. - /// - /// Note: currently a single local audio track is supported per peer + /// Callback invoked when a remote audio track is added to the peer /// connection. - virtual void MRS_API - SetLocalAudioTrackEnabled(bool enabled = true) noexcept = 0; + using AudioTrackAddedCallback = Callback; - /// Check if the local audio frame is enabled. - /// - /// Note: currently a single local audio track is supported per peer + /// Register a custom AudioTrackAddedCallback invoked when a remote audio + /// track is is added to the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterAudioTrackAddedCallback( + AudioTrackAddedCallback&& callback) noexcept = 0; + + /// Callback invoked when a remote audio track is removed from the peer /// connection. - virtual bool IsLocalAudioTrackEnabled() const noexcept = 0; + using AudioTrackRemovedCallback = Callback; + + /// Register a custom AudioTrackRemovedCallback invoked when a remote audio + /// track is removed from the peer connection. Only one callback can be + /// registered at a time. + virtual void RegisterAudioTrackRemovedCallback( + AudioTrackRemovedCallback&& callback) noexcept = 0; // // Data channel @@ -368,8 +372,7 @@ class PeerConnection : public TrackedObject { /// Close a given data channel and remove it from the peer connection. /// This invokes the DataChannelRemoved callback. - virtual void MRS_API - RemoveDataChannel(const DataChannel& data_channel) noexcept = 0; + virtual void RemoveDataChannel(const DataChannel& data_channel) noexcept = 0; /// Close and remove from the peer connection all data channels at once. /// This invokes the DataChannelRemoved callback for each data channel. @@ -378,8 +381,7 @@ class PeerConnection : public TrackedObject { /// Notification from a non-negotiated DataChannel that it is open, so that /// the PeerConnection can fire a DataChannelAdded event. This is called /// automatically by non-negotiated data channels; do not call manually. - virtual void MRS_API - OnDataChannelAdded(const DataChannel& data_channel) noexcept = 0; + virtual void OnDataChannelAdded(const DataChannel& data_channel) noexcept = 0; /// Internal use. void GetStats(webrtc::RTCStatsCollectorCallback* callback); @@ -394,6 +396,38 @@ class PeerConnection : public TrackedObject { virtual mrsResult RegisterInteropCallbacks( const mrsPeerConnectionInteropCallbacks& callbacks) noexcept = 0; + // + // Internal + // + + /// Internal callback on local audio track added to an audio transceiver of + /// this peer connection, to add it to the internal collection of local audio + /// tracks. + virtual void OnLocalTrackAddedToAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) = 0; + + /// Internal callback on local audio track removed from an audio transceiver + /// of this peer connection, to remove it from the internal collection of + /// local audio tracks. + virtual void OnLocalTrackRemovedFromAudioTransceiver( + AudioTransceiver& transceiver, + LocalAudioTrack& track) = 0; + + /// Internal callback on local video track added to a video transceiver of + /// this peer connection, to add it to the internal collection of local video + /// tracks. + virtual void OnLocalTrackAddedToVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) = 0; + + /// Internal callback on local video track removed from a video transceiver + /// of this peer connection, to remove it from the internal collection of + /// local video tracks. + virtual void OnLocalTrackRemovedFromVideoTransceiver( + VideoTransceiver& transceiver, + LocalVideoTrack& track) = 0; + protected: PeerConnection(RefPtr global_factory); }; diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h index 086022659..2ffb62ac2 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/tracked_object.h @@ -19,8 +19,14 @@ class GlobalFactory; /// factory while alive. enum class ObjectType : int { kPeerConnection, + kLocalAudioTrack, kLocalVideoTrack, kExternalVideoTrackSource, + kRemoteAudioTrack, + kRemoteVideoTrack, + kDataChannel, + kAudioTransceiver, + kVideoTransceiver, }; /// Object tracked for interop, exposing helper methods for debugging purpose. diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h b/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h index d6383a3d8..d8e6b8df5 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/utils.h @@ -1,30 +1,30 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#pragma once - -#include - -inline bool IsStringNullOrEmpty(const char* str) noexcept { - return ((str == nullptr) || (str[0] == '\0')); -} - -enum class mrsShutdownOptions : uint32_t; - -inline mrsShutdownOptions operator|(mrsShutdownOptions a, - mrsShutdownOptions b) noexcept { - return (mrsShutdownOptions)((uint32_t)a | (uint32_t)b); -} - -inline mrsShutdownOptions operator&(mrsShutdownOptions a, - mrsShutdownOptions b) noexcept { - return (mrsShutdownOptions)((uint32_t)a & (uint32_t)b); -} - -inline bool operator==(mrsShutdownOptions a, uint32_t b) noexcept { - return ((uint32_t)a == b); -} - -inline bool operator!=(mrsShutdownOptions a, uint32_t b) noexcept { - return ((uint32_t)a != b); -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include + +inline bool IsStringNullOrEmpty(const char* str) noexcept { + return ((str == nullptr) || (str[0] == '\0')); +} + +enum class mrsShutdownOptions : uint32_t; + +inline mrsShutdownOptions operator|(mrsShutdownOptions a, + mrsShutdownOptions b) noexcept { + return (mrsShutdownOptions)((uint32_t)a | (uint32_t)b); +} + +inline mrsShutdownOptions operator&(mrsShutdownOptions a, + mrsShutdownOptions b) noexcept { + return (mrsShutdownOptions)((uint32_t)a & (uint32_t)b); +} + +inline bool operator==(mrsShutdownOptions a, uint32_t b) noexcept { + return ((uint32_t)a == b); +} + +inline bool operator!=(mrsShutdownOptions a, uint32_t b) noexcept { + return ((uint32_t)a != b); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj index a32d684d8..923babe5b 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj @@ -133,8 +133,20 @@ + + + + + + + + + + + + @@ -149,6 +161,18 @@ + + + + + + + + + + + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters index 4bed116fb..71860cdee 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters @@ -30,6 +30,42 @@ media + + media + + + media + + + interop + + + interop + + + interop + + + media + + + media + + + interop + + + interop + + + media + + + media + + + media + @@ -59,6 +95,42 @@ media + + media + + + media + + + media + + + interop + + + interop + + + interop + + + media + + + interop + + + interop + + + media + + + media + + + media + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj index 264582862..d93717071 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj @@ -135,7 +135,13 @@ + + + + + + @@ -148,6 +154,12 @@ + + + + + + @@ -163,6 +175,18 @@ + + + + + + + + + + + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters index 2d42823e9..a9512744f 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters @@ -30,6 +30,42 @@ media + + interop + + + interop + + + interop + + + media + + + media + + + media + + + media + + + media + + + media + + + media + + + interop + + + interop + @@ -66,6 +102,42 @@ media + + interop + + + interop + + + interop + + + media + + + media + + + media + + + media + + + media + + + media + + + media + + + interop + + + interop + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj index b6b937709..d061865d4 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/Microsoft.MixedReality.WebRTC.Native.Tests.vcxproj @@ -58,11 +58,13 @@ + + @@ -73,8 +75,10 @@ + + diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp index 30de47ab4..18d2eb59d 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_track_tests.cpp @@ -4,6 +4,9 @@ #include "pch.h" #include "audio_frame.h" +#include "interop/audio_transceiver_interop.h" +#include "interop/local_audio_track_interop.h" +#include "interop/remote_audio_track_interop.h" #include "interop_api.h" #include "test_utils.h" @@ -18,6 +21,24 @@ class AudioTrackTests : public TestUtils::TestBase {}; namespace { +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; +const mrsRemoteAudioTrackInteropHandle kFakeInteropRemoteAudioTrackHandle = + (void*)0x2; + +mrsRemoteAudioTrackInteropHandle MRS_CALL FakeIterop_RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteAudioTrackHandle; +} + +// PeerConnectionAudioTrackAddedCallback +using AudioTrackAddedCallback = + InteropCallback; + // PeerConnectionAudioFrameCallback using AudioFrameCallback = InteropCallback; @@ -80,12 +101,112 @@ bool IsSilent_int16(const int16_t* data, TEST_F(AudioTrackTests, Simple) { LocalPeerPairRaii pair; - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddLocalAudioTrack(pair.pc1())); - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // AudioTrackAdded callback. + AudioTransceiverHandle audio_transceiver2{}; + RemoteAudioTrackHandle audio_track2{}; + Event track_added2_ev; + AudioTrackAddedCallback track_added2_cb = + [&audio_track2, &audio_transceiver2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteAudioTrackHandle track_handle, + mrsAudioTransceiverInteropHandle /*interop_handle*/, + AudioTransceiverHandle transceiver_handle) { + audio_track2 = track_handle; + audio_transceiver2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterAudioTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create an audio transceiver on #1 + AudioTransceiverHandle audio_transceiver1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "transceiver1"; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &audio_transceiver1)); + ASSERT_NE(nullptr, audio_transceiver1); + + // Create the local audio track #1 + LocalAudioTrackInitConfig config{}; + LocalAudioTrackHandle audio_track1{}; + ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( + &config, "test_audio_track", &audio_track1)); + ASSERT_NE(nullptr, audio_track1); + + // Audio tracks start enabled + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the local audio track on the transceiver #1 + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetLocalTrack(audio_transceiver1, audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is audio_track1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(audio_track1, track_handle_local); + mrsLocalAudioTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, audio_track2); + ASSERT_NE(nullptr, audio_transceiver2); + + // Check transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is audio_track2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver2, &track_handle_remote)); + ASSERT_EQ(audio_track2, track_handle_remote); + mrsRemoteAudioTrackRemoveRef(track_handle_remote); + } + // Register a callback on the remote track of #2 uint32_t call_count = 0; - AudioFrameCallback audio_cb = [&call_count](const AudioFrame& frame) { + AudioFrameCallback audio2_cb = [&call_count](const AudioFrame& frame) { ASSERT_NE(nullptr, frame.data_); ASSERT_LT(0u, frame.bits_per_sample_); ASSERT_LT(0u, frame.sampling_rate_hz_); @@ -111,15 +232,12 @@ TEST_F(AudioTrackTests, Simple) { //} ++call_count; }; - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), CB(audio_cb)); - - pair.ConnectAndWait(); + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, CB(audio2_cb)); // Check several times this, because the audio "mute" is flaky, does not // really mute the audio, so check that the reported status is still // correct. - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); // Give the track some time to stream audio data Event ev; @@ -127,24 +245,111 @@ TEST_F(AudioTrackTests, Simple) { ASSERT_LT(50u, call_count) << "Expected at least 10 CPS"; // Same as above - ASSERT_NE(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), nullptr, - nullptr); + // Clean-up + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); + mrsRemoteAudioTrackRemoveRef(audio_track2); + mrsAudioTransceiverRemoveRef(audio_transceiver2); + mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTransceiverRemoveRef(audio_transceiver1); } TEST_F(AudioTrackTests, Muted) { LocalPeerPairRaii pair; - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddLocalAudioTrack(pair.pc1())); + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // AudioTrackAdded callback. + AudioTransceiverHandle audio_transceiver2{}; + RemoteAudioTrackHandle audio_track2{}; + Event track_added2_ev; + AudioTrackAddedCallback track_added2_cb = + [&audio_track2, &audio_transceiver2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteAudioTrackHandle track_handle, + mrsAudioTransceiverInteropHandle /*interop_handle*/, + AudioTransceiverHandle transceiver_handle) { + audio_track2 = track_handle; + audio_transceiver2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterAudioTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create an audio transceiver on #1 + AudioTransceiverHandle audio_transceiver1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "transceiver1"; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &audio_transceiver1)); + ASSERT_NE(nullptr, audio_transceiver1); + + // Create the local audio track #1 + LocalAudioTrackInitConfig config{}; + LocalAudioTrackHandle audio_track1{}; + ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( + &config, "test_audio_track", &audio_track1)); + ASSERT_NE(nullptr, audio_track1); + + // Audio tracks start enabled + ASSERT_NE(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the local audio track on the transceiver #1 + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetLocalTrack(audio_transceiver1, audio_track1)); + + // Check transceiver #1 consistency + { + // Local track is audio_track1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + audio_transceiver1, &track_handle_local)); + ASSERT_EQ(audio_track1, track_handle_local); + mrsLocalAudioTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + audio_transceiver1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } // Disable the audio track; it should output only silence - ASSERT_EQ(Result::kSuccess, mrsPeerConnectionSetLocalAudioTrackEnabled( - pair.pc1(), mrsBool::kFalse)); - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(Result::kSuccess, + mrsLocalAudioTrackSetEnabled(audio_track1, mrsBool::kFalse)); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, audio_track2); + // Register a callback on the remote track of #2 uint32_t call_count = 0; AudioFrameCallback audio_cb = [&call_count, &pair](const AudioFrame& frame) { ASSERT_NE(nullptr, frame.data_); @@ -172,26 +377,26 @@ TEST_F(AudioTrackTests, Muted) { //} ++call_count; }; - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), CB(audio_cb)); - - pair.ConnectAndWait(); + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, CB(audio_cb)); // Check several times this, because the audio "mute" is flaky, does not // really mute the audio, so check that the reported status is still // correct. - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, call_count); // at least 10 CPS + ASSERT_LT(50u, call_count) << "Expected at least 10 CPS"; // Same as above - ASSERT_EQ(mrsBool::kFalse, - mrsPeerConnectionIsLocalAudioTrackEnabled(pair.pc1())); + ASSERT_EQ(mrsBool::kFalse, mrsLocalAudioTrackIsEnabled(audio_track1)); - mrsPeerConnectionRegisterRemoteAudioFrameCallback(pair.pc2(), nullptr, - nullptr); + // Clean-up + mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); + mrsRemoteAudioTrackRemoveRef(audio_track2); + mrsAudioTransceiverRemoveRef(audio_transceiver2); + mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTransceiverRemoveRef(audio_transceiver1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp new file mode 100644 index 000000000..1d7d44547 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/audio_transceiver_tests.cpp @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/audio_transceiver_interop.h" +#include "interop_api.h" +#include "interop/local_audio_track_interop.h" +#include "interop/remote_audio_track_interop.h" + +#include "simple_interop.h" + +namespace { + +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteAudioTrackInteropHandle kFakeInteropRemoteAudioTrackHandle = + (void*)0x2; + +const mrsAudioTransceiverInteropHandle kFakeInteropAudioTransceiverHandle = + (void*)0x3; + +/// Fake interop callback always returning the same fake remote audio track +/// interop handle, for tests which do not care about it. +mrsRemoteAudioTrackInteropHandle MRS_CALL FakeIterop_RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteAudioTrackHandle; +} + +// PeerConnectionAudioTrackAddedCallback +using AudioTrackAddedCallback = + InteropCallback; + +// PeerConnectionI420AudioFrameCallback +using I420AudioFrameCallback = InteropCallback; + +} // namespace + +TEST(AudioTransceiver, InvalidName) { + LocalPeerPairRaii pair; + AudioTransceiverHandle transceiver_handle1{}; + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "invalid name with space"; + ASSERT_EQ(Result::kInvalidParameter, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_EQ(nullptr, transceiver_handle1); +} + +TEST(AudioTransceiver, SetDirection) { + LocalPeerPairRaii pair; + + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &FakeIterop_RemoteAudioTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc1(), &interop)); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + AudioTransceiverHandle transceiver_handle1{}; + { + AudioTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "audio_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropAudioTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddAudioTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsAudioTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Check audio transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kInactive, dir_desired1); + + // Local audio track is NULL + LocalAudioTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote audio track is NULL + RemoteAudioTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Because the state updated event handler is registered after the transceiver + // is created, the state is stale, and applying the local description during + // |CreateOffer()| will generate some event. + ASSERT_TRUE(state_updated1_ev_local.WaitFor(10s)); + state_updated1_ev_local.Reset(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Set transceiver #1 direction to Receive + ASSERT_EQ(Result::kSuccess, + mrsAudioTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.IsSignaled()); + state_updated1_ev_setdir.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Send+Receive + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, + dir_negotiated1); // no change + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, during + // SetRemoteDescription() + // Note: here the local description doesn't generate a state updated event + // because the local state was set with SetDirection() so is already correct. + // When the peer is creating the offer (#1), the desired direction is exactly + // the one advertized in the local description. + ASSERT_FALSE(state_updated1_ev_local.IsSignaled()); + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check audio transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive because remote peer + // refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsAudioTransceiverRemoveRef(transceiver_handle1); +} + +TEST(AudioTransceiver, SetDirection_InvalidHandle) { + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsAudioTransceiverSetDirection( + nullptr, mrsTransceiverDirection::kRecvOnly)); +} + +TEST(AudioTransceiver, SetLocalTrack_InvalidHandle) { + LocalAudioTrackHandle dummy = (void*)0x1; // looks legit + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsAudioTransceiverSetLocalTrack(nullptr, dummy)); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp index f81f94c7b..39427a884 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/external_video_track_source_tests.cpp @@ -7,6 +7,8 @@ #include "external_video_track_source_interop.h" #include "interop_api.h" #include "local_video_track_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" #include "test_utils.h" @@ -112,6 +114,13 @@ void ValidateQuadTestFrame(const void* data, ASSERT_LE(std::fabs(err), 768.0); // +/-1 per component over 256 pixels } +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + // mrsArgb32VideoFrameCallback using Argb32VideoFrameCallback = InteropCallback; @@ -120,22 +129,114 @@ using Argb32VideoFrameCallback = InteropCallback; TEST_F(ExternalVideoTrackSourceTests, Simple) { LocalPeerPairRaii pair; - ExternalVideoTrackSourceHandle source_handle = nullptr; + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_handle) { + track_handle2 = track_handle; + transceiver_handle2 = transceiver_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; ASSERT_EQ(mrsResult::kSuccess, mrsExternalVideoTrackSourceCreateFromArgb32Callback( - &GenerateQuadTestFrame, nullptr, &source_handle)); - ASSERT_NE(nullptr, source_handle); - mrsExternalVideoTrackSourceFinishCreation(source_handle); + &GenerateQuadTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); - LocalVideoTrackHandle track_handle = nullptr; - LocalVideoTrackFromExternalSourceInitConfig source_config{}; - ASSERT_EQ(mrsResult::kSuccess, - mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - pair.pc1(), "gen_track", source_handle, &source_config, - &track_handle)); - ASSERT_NE(nullptr, track_handle); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + // Create the local track itself for #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig source_config{}; + ASSERT_EQ(mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &source_config, "gen_track", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(mrsResult::kSuccess, + mrsPeerConnectionAddVideoTransceiver(pair.pc1(), &config, + &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Check video transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Add the track #1 to the transceiver #1 + ASSERT_EQ(mrsResult::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + + // Check video transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is track_handle2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle2, &track_handle_remote)); + ASSERT_EQ(track_handle2, track_handle_remote); + mrsRemoteVideoTrackRemoveRef(track_handle_remote); + } + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; Argb32VideoFrameCallback argb_cb = [&frame_count](const mrsArgb32VideoFrame& frame) { @@ -146,22 +247,22 @@ TEST_F(ExternalVideoTrackSourceTests, Simple) { frame.height_); ++frame_count; }; - mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback(pair.pc2(), - CB(argb_cb)); + mrsRemoteVideoTrackRegisterArgb32FrameCallback(track_handle2, CB(argb_cb)); - pair.ConnectAndWait(); - - // Simple timer + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsPeerConnectionRemoveLocalVideoTracksFromSource(pair.pc1(), source_handle); - mrsLocalVideoTrackRemoveRef(track_handle); - mrsExternalVideoTrackSourceShutdown(source_handle); - mrsExternalVideoTrackSourceRemoveRef(source_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterArgb32FrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); + mrsExternalVideoTrackSourceShutdown(source_handle1); + mrsExternalVideoTrackSourceRemoveRef(source_handle1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h index a0e0ff6cb..81cf9d3db 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/pch.h @@ -40,6 +40,8 @@ using namespace std::chrono_literals; #pragma warning(push) #pragma warning(disable : 4100 4127) #include "api/datachannelinterface.h" +#include "api/peerconnectioninterface.h" +#include "rtc_base/memory/aligned_malloc.h" #include "rtc_base/thread_annotations.h" #pragma warning(pop) diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h index 9cca53b73..382e2a206 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_test_helpers.h @@ -76,6 +76,11 @@ struct Semaphore { int64_t value_{0}; }; +/// RAII helper to initialize and shutdown the library. +struct LibraryInitRaii { + // TODO - remove this +}; + /// Wrapper around an interop callback taking an extra raw pointer argument, to /// trampoline its call to a generic std::function for convenience (including /// lambda functions). diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp index 100d3f6e1..8e1f06ffe 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/peer_connection_tests.cpp @@ -32,8 +32,7 @@ TEST(PeerConnection, LocalNoIce) { pc2.handle(), type, sdp_data, &SetEventOnCompleted, &ev)); ev.Wait(); if (kOfferString == type) { - ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionCreateAnswer(pc2.handle())); + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionCreateAnswer(pc2.handle())); } }); SdpCallback sdp2_cb(pc2.handle(), [&pc1](const char* type, diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp new file mode 100644 index 000000000..1d651a567 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.cpp @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "simple_interop.h" + +void SimpleInterop::Register(PeerConnectionHandle pc) noexcept { + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_audio_track_create_object = &RemoteAudioTrackCreateStatic; + interop.remote_video_track_create_object = &RemoteVideoTrackCreateStatic; + interop.data_channel_create_object = &DataChannelCreateStatic; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pc, &interop)); +} + +void SimpleInterop::Unregister(PeerConnectionHandle pc) noexcept { + mrsPeerConnectionInteropCallbacks interop{}; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pc, &interop)); +} + +void* SimpleInterop::CreateObject(ObjectType type) noexcept { + const uint32_t id = free_id_.fetch_add(1); + auto handle = std::make_unique(Handle{this, type, id}); + void* interop_handle = handle.get(); + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + objects_map_.emplace(interop_handle, std::move(handle)); + } catch (...) { + } + return interop_handle; +} + +void SimpleInterop::DestroyObject(void* obj) noexcept { + const uint32_t id = (uint32_t) reinterpret_cast(obj); + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + auto it = objects_map_.find(obj); + EXPECT_NE(objects_map_.end(), it); + if (it != objects_map_.end()) { + EXPECT_EQ(this, it->second->interop_); + EXPECT_EQ(id, it->second->id_); + objects_map_.erase(it); + } + } catch (...) { + } +} + +bool SimpleInterop::ObjectExists(ObjectType type, void* obj) noexcept { + try { + auto lock = std::scoped_lock{objects_map_mutex_}; + auto it = objects_map_.find(obj); + if (it != objects_map_.end()) { + // If an object address is found, it must be registered with the current + // interop and with the right object type. + EXPECT_EQ(this, it->second->interop_); + EXPECT_EQ(type, it->second->object_type_); + return true; + } + } catch (...) { + } + return false; +} + +mrsRemoteAudioTrackInteropHandle SimpleInterop::RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& /*config*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kRemoteAudioTrack); +} + +mrsRemoteVideoTrackInteropHandle SimpleInterop::RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kRemoteVideoTrack); +} + +mrsDataChannelInteropHandle SimpleInterop::DataChannelCreate( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& /*config*/, + mrsDataChannelCallbacks* /*callbacks*/) noexcept { + EXPECT_TRUE(ObjectExists(ObjectType::kPeerConnection, parent)); + return CreateObject(ObjectType::kDataChannel); +} + +mrsRemoteAudioTrackInteropHandle MRS_CALL +SimpleInterop::RemoteAudioTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + return parent_handle->interop_->RemoteAudioTrackCreate(parent, config); +} + +mrsRemoteVideoTrackInteropHandle MRS_CALL +SimpleInterop::RemoteVideoTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + return parent_handle->interop_->RemoteVideoTrackCreate(parent, config); +} + +mrsDataChannelInteropHandle MRS_CALL SimpleInterop::DataChannelCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept { + auto parent_handle = (Handle*)parent; + EXPECT_NE(nullptr, parent_handle); + EXPECT_NE(nullptr, callbacks); + return parent_handle->interop_->DataChannelCreate(parent, config, callbacks); +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h new file mode 100644 index 000000000..37ec4491f --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/simple_interop.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "../src/mrs_errors.h" +#include "../src/interop/global_factory.h" +#include "../include/interop_api.h" +#include "../include/peer_connection_interop.h" + +using namespace Microsoft::MixedReality::WebRTC; + +/// A simple interop layer maintaining a hash map of all objects, with assigned +/// unique identifiers. Use this to keep track of multiple interop objects in +/// tests and ensure consistency of interop handle types. +class SimpleInterop { + public: + using ObjectType = Microsoft::MixedReality::WebRTC::ObjectType; + + struct Handle { + SimpleInterop* interop_; + ObjectType object_type_; + uint32_t id_; + }; + + void Register(PeerConnectionHandle pc) noexcept; + void Unregister(PeerConnectionHandle pc) noexcept; + + void* CreateObject(ObjectType type) noexcept; + void DestroyObject(void* obj) noexcept; + bool ObjectExists(ObjectType type, void* obj) noexcept; + + mrsRemoteAudioTrackInteropHandle RemoteAudioTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + mrsRemoteVideoTrackInteropHandle RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + mrsDataChannelInteropHandle DataChannelCreate( + mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept; + + static mrsRemoteAudioTrackInteropHandle MRS_CALL RemoteAudioTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteAudioTrackConfig& config) noexcept; + static mrsRemoteVideoTrackInteropHandle MRS_CALL RemoteVideoTrackCreateStatic( + mrsPeerConnectionInteropHandle parent, + const mrsRemoteVideoTrackConfig& config) noexcept; + static mrsDataChannelInteropHandle MRS_CALL + DataChannelCreateStatic(mrsPeerConnectionInteropHandle parent, + const mrsDataChannelConfig& config, + mrsDataChannelCallbacks* callbacks) noexcept; + + std::mutex objects_map_mutex_; + std::unordered_map> objects_map_; + std::atomic_uint32_t free_id_; +}; diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h index 5f35f32e6..202ad970e 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_test_utils.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "../src/mrs_errors.h" +#include "../src/interop/global_factory.h" #include "../include/interop_api.h" #include "../include/peer_connection_interop.h" #include "../src/mrs_errors.h" @@ -16,4 +18,4 @@ mrsResult MRS_CALL MakeTestFrame(void* /*user_data*/, void CheckIsTestFrame(const I420AVideoFrame& frame); -} // namespace VideoTestUtils +} diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp index d81a00751..c30693b46 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_track_tests.cpp @@ -4,9 +4,12 @@ #include "pch.h" #include "external_video_track_source_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" #include "interop_api.h" #include "local_video_track_interop.h" +#include "simple_interop.h" #include "test_utils.h" #include "video_test_utils.h" @@ -14,6 +17,27 @@ namespace { +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteVideoTrackInteropHandle kFakeInteropRemoteVideoTrackHandle = + (void*)0x2; + +/// Fake interop callback always returning the same fake remote video track +/// interop handle, for tests which do not care about it. +mrsRemoteVideoTrackInteropHandle MRS_CALL FakeIterop_RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteVideoTrackHandle; +} + +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + // PeerConnectionI420VideoFrameCallback using I420VideoFrameCallback = InteropCallback; @@ -24,13 +48,129 @@ class VideoTrackTests : public TestUtils::TestBase {}; TEST_F(VideoTrackTests, Simple) { LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; - LocalVideoTrackHandle track_handle{}; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "local_video_track", - &config, &track_handle)); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + renegotiation_needed1_ev.Reset(); + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.WaitFor(1s)); + renegotiation_needed1_ev.Reset(); + } + + // Check video transceiver #1 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Create the local video track #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackInitConfig config{}; + ASSERT_EQ(Result::kSuccess, + mrsLocalVideoTrackCreateFromDevice(&config, "local_video_track", + &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + } + + // New tracks are enabled by default + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + + // Add the local track #1 on the transceiver #1. + ASSERT_FALSE(renegotiation_needed1_ev.IsSignaled()); + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + ASSERT_FALSE( + renegotiation_needed1_ev + .IsSignaled()); // TODO: why? because transceiver starts in SendRecv + // mode, so didn't change direction? + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + + // Check video transceiver #2 consistency + { + // Local track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle2, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote track is track_handle2 + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle2, &track_handle_remote)); + ASSERT_EQ(track_handle2, track_handle_remote); + mrsRemoteVideoTrackRemoveRef(track_handle_remote); + } + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { ASSERT_NE(nullptr, frame.ydata_); @@ -40,37 +180,91 @@ TEST_F(VideoTrackTests, Simple) { ASSERT_LT(0u, frame.height_); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsLocalVideoTrackRemoveRef(track_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); } TEST_F(VideoTrackTests, Muted) { LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; - LocalVideoTrackHandle track_handle{}; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; ASSERT_EQ(Result::kSuccess, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "local_video_track", - &config, &track_handle)); + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Create the local video track #1 + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackInitConfig config{}; + ASSERT_EQ(Result::kSuccess, + mrsLocalVideoTrackCreateFromDevice(&config, "local_video_track", + &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + } // New tracks are enabled by default - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); // Disable the video track; it should output only black frames ASSERT_EQ(Result::kSuccess, - mrsLocalVideoTrackSetEnabled(track_handle, mrsBool::kFalse)); - ASSERT_EQ(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + mrsLocalVideoTrackSetEnabled(track_handle1, mrsBool::kFalse)); + ASSERT_EQ(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + // Add the local track #1 on the transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { ASSERT_NE(nullptr, frame.ydata_); @@ -88,18 +282,20 @@ TEST_F(VideoTrackTests, Muted) { ASSERT_TRUE(all_black); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); + // Wait 5 seconds and check the frame callback is called Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsLocalVideoTrackRemoveRef(track_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + // Clean-up + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); } void MRS_CALL enumDeviceCallback(const char* id, @@ -115,7 +311,7 @@ void MRS_CALL enumDeviceCallbackCompleted(void* user_data) { } // FIXME - PeerConnection currently doesn't support multiple local video tracks -// TEST(VideoTrack, DeviceIdAll) { +// TEST_F(VideoTrackTests, DeviceIdAll) { // LocalPeerPairRaii pair; // // Event ev; @@ -133,56 +329,264 @@ void MRS_CALL enumDeviceCallbackCompleted(void* user_data) { //} TEST_F(VideoTrackTests, DeviceIdInvalid) { - LocalPeerPairRaii pair; - LocalVideoTrackInitConfig config{}; LocalVideoTrackHandle track_handle{}; config.video_device_id = "[[INVALID DEVICE ID]]"; - ASSERT_EQ(Result::kNotFound, - mrsPeerConnectionAddLocalVideoTrack(pair.pc1(), "invalid_track", - &config, &track_handle)); + ASSERT_EQ(Result::kNotFound, mrsLocalVideoTrackCreateFromDevice( + &config, "invalid_track", &track_handle)); ASSERT_EQ(nullptr, track_handle); } +TEST_F(VideoTrackTests, Multi) { + SimpleInterop simple_interop1; + SimpleInterop simple_interop2; + + mrsPeerConnectionInteropHandle h1 = + simple_interop1.CreateObject(ObjectType::kPeerConnection); + mrsPeerConnectionInteropHandle h2 = + simple_interop2.CreateObject(ObjectType::kPeerConnection); + + PeerConnectionConfiguration pc_config{}; + LocalPeerPairRaii pair(pc_config, h1, h2); + + constexpr const int kNumTracks = 5; + struct TestTrack { + int id{0}; + int frame_count{0}; + I420VideoFrameCallback frame_cb{}; + LocalVideoTrackHandle local_handle{}; + RemoteVideoTrackHandle remote_handle{}; + VideoTransceiverHandle local_transceiver_handle{}; + VideoTransceiverHandle remote_transceiver_handle{}; + }; + TestTrack tracks[kNumTracks]; + + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + simple_interop1.Register(pair.pc1()); + simple_interop2.Register(pair.pc2()); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + Semaphore track_added2_sem; + std::atomic_int32_t track_id{0}; + VideoTrackAddedCallback track_added2_cb = + [&track_added2_sem, &track_id, &tracks, kNumTracks]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_handle) { + int id = track_id.fetch_add(1); + ASSERT_LT(id, kNumTracks); + tracks[id].remote_handle = track_handle; + tracks[id].remote_transceiver_handle = transceiver_handle; + track_added2_sem.Release(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the external source for the local tracks of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create local video tracks on the local peer (#1) + LocalVideoTrackFromExternalSourceInitConfig track_config{}; + int idx = 0; + for (auto&& track : tracks) { + std::stringstream strstr; + std::string str; + VideoTransceiverInitConfig tranceiver_config{}; + strstr << "transceiver_1_" << idx; + str = strstr.str(); // keep alive + tranceiver_config.name = str.c_str(); + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &tranceiver_config, + &track.local_transceiver_handle)); + ASSERT_NE(nullptr, track.local_transceiver_handle); + strstr.clear(); + strstr << "track_1_" << idx; + str = strstr.str(); // keep alive + ASSERT_EQ(Result::kSuccess, mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &track_config, str.c_str(), + &track.local_handle)); + ASSERT_NE(nullptr, track.local_handle); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(track.local_transceiver_handle, + track.local_handle)); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track.local_handle)); + + // Check video transceiver consistency + { + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverGetLocalTrack(track.local_transceiver_handle, + &track_handle_local)); + ASSERT_EQ(track.local_handle, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverGetRemoteTrack( + track.local_transceiver_handle, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + ++idx; + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for all remote tracks to be added on #2 + ASSERT_TRUE(track_added2_sem.TryAcquireFor(5s, kNumTracks)); + for (auto&& track : tracks) { + ASSERT_NE(nullptr, track.remote_handle); + } + + // Register a frame callback for the remote video of #2 + for (auto&& track : tracks) { + track.frame_cb = [&track](const I420AVideoFrame& frame) { + ASSERT_NE(nullptr, frame.ydata_); + ASSERT_NE(nullptr, frame.udata_); + ASSERT_NE(nullptr, frame.vdata_); + ASSERT_LT(0u, frame.width_); + ASSERT_LT(0u, frame.height_); + ++track.frame_count; + }; + mrsRemoteVideoTrackRegisterI420AFrameCallback(track.remote_handle, + CB(track.frame_cb)); + } + + Event ev; + ev.WaitFor(5s); + for (auto&& track : tracks) { + ASSERT_LT(50, track.frame_count) << "Expected at least 10 FPS"; + } + + // Clean-up + for (auto&& track : tracks) { + mrsRemoteVideoTrackRegisterI420AFrameCallback(track.remote_handle, nullptr, + nullptr); + mrsRemoteVideoTrackRemoveRef(track.remote_handle); + mrsVideoTransceiverRemoveRef(track.remote_transceiver_handle); + mrsLocalVideoTrackRemoveRef(track.local_handle); + mrsVideoTransceiverRemoveRef(track.local_transceiver_handle); + } + + simple_interop1.Unregister(pair.pc1()); + simple_interop2.Unregister(pair.pc2()); +} + TEST_F(VideoTrackTests, ExternalI420) { LocalPeerPairRaii pair; - ExternalVideoTrackSourceHandle source_handle = nullptr; + // In order to allow creating interop wrappers from native code, register the + // necessary interop callbacks. + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = &FakeIterop_RemoteVideoTrackCreate; + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(pair.pc2(), &interop)); + + // Grab the handle of the remote track from the remote peer (#2) via the + // VideoTrackAdded callback. + RemoteVideoTrackHandle track_handle2{}; + VideoTransceiverHandle transceiver_handle2{}; + Event track_added2_ev; + VideoTrackAddedCallback track_added2_cb = + [&track_handle2, &transceiver_handle2, &track_added2_ev]( + mrsRemoteVideoTrackInteropHandle /*interop_handle*/, + RemoteVideoTrackHandle track_native_handle, + mrsVideoTransceiverInteropHandle /*interop_handle*/, + VideoTransceiverHandle transceiver_native_handle) { + track_handle2 = track_native_handle; + transceiver_handle2 = transceiver_native_handle; + track_added2_ev.Set(); + }; + mrsPeerConnectionRegisterVideoTrackAddedCallback(pair.pc2(), + CB(track_added2_cb)); + + // Create the video transceiver #1 + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig config{}; + config.name = "transceiver_1"; + ASSERT_EQ(Result::kSuccess, mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; ASSERT_EQ(mrsResult::kSuccess, mrsExternalVideoTrackSourceCreateFromI420ACallback( - &VideoTestUtils::MakeTestFrame, nullptr, &source_handle)); - ASSERT_NE(nullptr, source_handle); - mrsExternalVideoTrackSourceFinishCreation(source_handle); + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add the local track #1 on the transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Local track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); - LocalVideoTrackHandle track_handle = nullptr; - LocalVideoTrackFromExternalSourceInitConfig source_config{}; - ASSERT_EQ(mrsResult::kSuccess, - mrsPeerConnectionAddLocalVideoTrackFromExternalSource( - pair.pc1(), "simulated_video_track", source_handle, - &source_config, &track_handle)); - ASSERT_NE(nullptr, track_handle); - ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle)); + // Wait for remote track to be added on #2 + ASSERT_TRUE(track_added2_ev.WaitFor(5s)); + ASSERT_NE(nullptr, track_handle2); + ASSERT_NE(nullptr, transceiver_handle2); + // Register a frame callback for the remote video of #2 uint32_t frame_count = 0; I420VideoFrameCallback i420cb = [&frame_count](const I420AVideoFrame& frame) { VideoTestUtils::CheckIsTestFrame(frame); ++frame_count; }; - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), - CB(i420cb)); - - pair.ConnectAndWait(); + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, CB(i420cb)); Event ev; ev.WaitFor(5s); - ASSERT_LT(50u, frame_count); // at least 10 FPS - - mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback(pair.pc2(), nullptr, - nullptr); - mrsPeerConnectionRemoveLocalVideoTrack(pair.pc1(), track_handle); - mrsLocalVideoTrackRemoveRef(track_handle); - mrsExternalVideoTrackSourceShutdown(source_handle); - mrsExternalVideoTrackSourceRemoveRef(source_handle); + ASSERT_LT(50u, frame_count) << "Expected at least 10 FPS"; + + mrsRemoteVideoTrackRegisterI420AFrameCallback(track_handle2, nullptr, + nullptr); + mrsLocalVideoTrackRemoveRef(track_handle1); + mrsVideoTransceiverRemoveRef(transceiver_handle1); + mrsRemoteVideoTrackRemoveRef(track_handle2); + mrsVideoTransceiverRemoveRef(transceiver_handle2); + mrsExternalVideoTrackSourceShutdown(source_handle1); + mrsExternalVideoTrackSourceRemoveRef(source_handle1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp new file mode 100644 index 000000000..83deca12e --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC.Native/test/video_transceiver_tests.cpp @@ -0,0 +1,612 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "external_video_track_source_interop.h" +#include "interop_api.h" +#include "local_video_track_interop.h" +#include "interop/remote_video_track_interop.h" +#include "interop/video_transceiver_interop.h" + +#include "simple_interop.h" +#include "video_test_utils.h" + +namespace { + +const mrsPeerConnectionInteropHandle kFakeInteropPeerConnectionHandle = + (void*)0x1; + +const mrsRemoteVideoTrackInteropHandle kFakeInteropRemoteVideoTrackHandle = + (void*)0x2; + +const mrsVideoTransceiverInteropHandle kFakeInteropVideoTransceiverHandle = + (void*)0x3; + +/// Fake interop callback always returning the same fake remote video track +/// interop handle, for tests which do not care about it. +mrsRemoteVideoTrackInteropHandle MRS_CALL FakeIterop_RemoteVideoTrackCreate( + mrsPeerConnectionInteropHandle /*parent*/, + const mrsRemoteVideoTrackConfig& /*config*/) noexcept { + return kFakeInteropRemoteVideoTrackHandle; +} + +struct FakeInteropRaii { + FakeInteropRaii(std::initializer_list handles) + : handles_(handles) { + setup(); + } + ~FakeInteropRaii() { cleanup(); } + void setup() { + mrsPeerConnectionInteropCallbacks interop{}; + interop.remote_video_track_create_object = + &FakeIterop_RemoteVideoTrackCreate; + for (auto&& h : handles_) { + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionRegisterInteropCallbacks(h, &interop)); + } + } + void cleanup() {} + std::vector handles_; +}; + +// PeerConnectionVideoTrackAddedCallback +using VideoTrackAddedCallback = + InteropCallback; + +// PeerConnectionI420VideoFrameCallback +using I420VideoFrameCallback = InteropCallback; + +} // namespace + +TEST(VideoTransceiver, InvalidName) { + LocalPeerPairRaii pair; + VideoTransceiverHandle transceiver_handle1{}; + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "invalid name with space"; + ASSERT_EQ(Result::kInvalidParameter, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_EQ(nullptr, transceiver_handle1); +} + +TEST(VideoTransceiver, SetDirection) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kInactive, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Because the state updated event handler is registered after the transceiver + // is created, the state is stale, and applying the local description during + // |CreateOffer()| will generate some event. + ASSERT_TRUE(state_updated1_ev_local.WaitFor(10s)); + state_updated1_ev_local.Reset(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Set transceiver #1 direction to Receive + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.IsSignaled()); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Send+Receive + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, + dir_negotiated1); // no change + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, during + // SetRemoteDescription() + // Note: here the local description doesn't generate a state updated event + // because the local state was set with SetDirection() so is already correct. + // When the peer is creating the offer (#1), the desired direction is exactly + // the one advertized in the local description. + ASSERT_FALSE(state_updated1_ev_local.IsSignaled()); + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive because remote peer + // refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetDirection_InvalidHandle) { + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsVideoTransceiverSetDirection( + nullptr, mrsTransceiverDirection::kRecvOnly)); +} + +TEST(VideoTransceiver, SetLocalTrackSendRecv) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Start in Send+Receive mode for this test + state_updated1_ev_setdir.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kSendRecv)); + ASSERT_TRUE(state_updated1_ev_setdir.WaitFor(10s)); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is Send only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track1", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add track to transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is still Send only. + // SetLocalTrack() doesn't change the transceiver directions. + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Remove track from transceiver #1 with non-null track + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(transceiver_handle1, nullptr)); + + // Check video transceiver #1 consistency + { + // Desired state is Send+Receive, negotiated is still Send only. + // SetLocalTrack() doesn't change the transceiver directions. + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Check video transceiver #1 consistency + //< FIXME - In theory should wait for SetRemoteDesc on #1 (from #2's answer), + // but since state doesn't change there is no way to wait for that. + { + // Again, nothing changed + ASSERT_EQ(mrsTransceiverOptDirection::kSendOnly, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kSendRecv, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetLocalTrackRecvOnly) { + LocalPeerPairRaii pair; + FakeInteropRaii interop({pair.pc1(), pair.pc2()}); + + // Register event for renegotiation needed + Event renegotiation_needed1_ev; + InteropCallback renegotiation_needed1_cb = [&renegotiation_needed1_ev]() { + renegotiation_needed1_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc1(), CB(renegotiation_needed1_cb)); + Event renegotiation_needed2_ev; + InteropCallback renegotiation_needed2_cb = [&renegotiation_needed2_ev]() { + renegotiation_needed2_ev.Set(); + }; + mrsPeerConnectionRegisterRenegotiationNeededCallback( + pair.pc2(), CB(renegotiation_needed2_cb)); + + // Add a transceiver to the local peer (#1) + VideoTransceiverHandle transceiver_handle1{}; + { + VideoTransceiverInitConfig transceiver_config{}; + transceiver_config.name = "video_transceiver_1"; + transceiver_config.transceiver_interop_handle = + kFakeInteropVideoTransceiverHandle; + renegotiation_needed1_ev.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsPeerConnectionAddVideoTransceiver( + pair.pc1(), &transceiver_config, &transceiver_handle1)); + ASSERT_NE(nullptr, transceiver_handle1); + ASSERT_TRUE(renegotiation_needed1_ev.IsSignaled()); + renegotiation_needed1_ev.Reset(); + } + + // Register event for transceiver state update + Event state_updated1_ev_local; + Event state_updated1_ev_remote; + Event state_updated1_ev_setdir; + mrsTransceiverDirection dir_desired1 = mrsTransceiverDirection::kInactive; + mrsTransceiverOptDirection dir_negotiated1 = + mrsTransceiverOptDirection::kNotSet; + InteropCallback + state_updated1_cb = [&](mrsTransceiverStateUpdatedReason reason, + mrsTransceiverOptDirection negotiated, + mrsTransceiverDirection desired) { + dir_negotiated1 = negotiated; + dir_desired1 = desired; + switch (reason) { + case mrsTransceiverStateUpdatedReason::kLocalDesc: + state_updated1_ev_local.Set(); + break; + case mrsTransceiverStateUpdatedReason::kRemoteDesc: + state_updated1_ev_remote.Set(); + break; + case mrsTransceiverStateUpdatedReason::kSetDirection: + state_updated1_ev_setdir.Set(); + break; + } + }; + mrsVideoTransceiverRegisterStateUpdatedCallback(transceiver_handle1, + CB(state_updated1_cb)); + + // Start in receive mode for this test + state_updated1_ev_setdir.Reset(); + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetDirection( + transceiver_handle1, mrsTransceiverDirection::kRecvOnly)); + ASSERT_TRUE(state_updated1_ev_setdir.WaitFor(10s)); + state_updated1_ev_setdir.Reset(); + + // Check video transceiver #1 consistency + { + // Default values inchanged (callback was just registered) + ASSERT_EQ(mrsTransceiverOptDirection::kNotSet, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Connect #1 and #2 + pair.ConnectAndWait(); + + // Wait for transceiver to be updated; this happens *after* connect, + // during SetRemoteDescription(). + ASSERT_TRUE(state_updated1_ev_remote.WaitFor(10s)); + state_updated1_ev_remote.Reset(); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is Inactive only because the remote + // peer refused to send (no track added for that). + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Create the external source for the local video track of the local peer (#1) + ExternalVideoTrackSourceHandle source_handle1 = nullptr; + ASSERT_EQ(mrsResult::kSuccess, + mrsExternalVideoTrackSourceCreateFromI420ACallback( + &VideoTestUtils::MakeTestFrame, nullptr, &source_handle1)); + ASSERT_NE(nullptr, source_handle1); + mrsExternalVideoTrackSourceFinishCreation(source_handle1); + + // Create the local video track (#1) + LocalVideoTrackHandle track_handle1{}; + { + LocalVideoTrackFromExternalSourceInitConfig config{}; + ASSERT_EQ( + mrsResult::kSuccess, + mrsLocalVideoTrackCreateFromExternalSource( + source_handle1, &config, "simulated_video_track1", &track_handle1)); + ASSERT_NE(nullptr, track_handle1); + ASSERT_NE(mrsBool::kFalse, mrsLocalVideoTrackIsEnabled(track_handle1)); + } + + // Add track to transceiver #1 + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverSetLocalTrack( + transceiver_handle1, track_handle1)); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Inactive + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is track_handle1 + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(track_handle1, track_handle_local); + mrsLocalVideoTrackRemoveRef(track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Remote track from transceiver #1 with non-null track + ASSERT_EQ(Result::kSuccess, + mrsVideoTransceiverSetLocalTrack(transceiver_handle1, nullptr)); + + // Check video transceiver #1 consistency + { + // Desired state is Receive, negotiated is still Inactive + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + + // Local video track is NULL + LocalVideoTrackHandle track_handle_local{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetLocalTrack( + transceiver_handle1, &track_handle_local)); + ASSERT_EQ(nullptr, track_handle_local); + + // Remote video track is NULL + RemoteVideoTrackHandle track_handle_remote{}; + ASSERT_EQ(Result::kSuccess, mrsVideoTransceiverGetRemoteTrack( + transceiver_handle1, &track_handle_remote)); + ASSERT_EQ(nullptr, track_handle_remote); + } + + // Renegotiate + pair.ConnectAndWait(); + + // Check video transceiver #1 consistency + //< FIXME - In theory should wait for SetRemoteDesc on #1 (from #2's answer), + // but since state doesn't change there is no way to wait for that. + { + // Note how nothing changed, because SetLocalTrack() does not change the + // desired direction of Receive, and the remote peer #2 still doesn't have a + // track to send us. + ASSERT_EQ(mrsTransceiverOptDirection::kInactive, dir_negotiated1); + ASSERT_EQ(mrsTransceiverDirection::kRecvOnly, dir_desired1); + } + + // Clean-up + mrsVideoTransceiverRemoveRef(transceiver_handle1); +} + +TEST(VideoTransceiver, SetLocalTrack_InvalidHandle) { + LocalVideoTrackHandle dummy = (void*)0x1; // looks legit + ASSERT_EQ(Result::kInvalidNativeHandle, + mrsVideoTransceiverSetLocalTrack(nullptr, dummy)); +}