diff --git a/examples/TestAppUwp/MainPage.xaml.cs b/examples/TestAppUwp/MainPage.xaml.cs index 635cff840..2c4719de2 100644 --- a/examples/TestAppUwp/MainPage.xaml.cs +++ b/examples/TestAppUwp/MainPage.xaml.cs @@ -66,25 +66,29 @@ public sealed partial class MainPage : Page /// public bool PluginInitialized { get; private set; } = false; + private Transceiver _audioTransceiver = null; + private Transceiver _videoTransceiver = null; + private MediaStreamSource localVideoSource = null; private MediaSource localMediaSource = null; private MediaPlayer localVideoPlayer = new MediaPlayer(); private bool _isLocalVideoPlaying = false; - private object _isLocalVideoPlayingLock = new object(); + private object _isLocalMediaPlayingLock = new object(); + private LocalAudioTrack _localAudioTrack = null; private LocalVideoTrack _localVideoTrack = null; private MediaStreamSource remoteVideoSource = null; private MediaSource remoteMediaSource = null; private MediaPlayer remoteVideoPlayer = new MediaPlayer(); private bool _isRemoteVideoPlaying = false; + private bool _isRemoteAudioPlaying = false; private uint _remoteVideoWidth = 0; private uint _remoteVideoHeight = 0; - private object _isRemoteVideoPlayingLock = new object(); - private uint _remoteAudioChannelCount = 0; private uint _remoteAudioSampleRate = 0; - private bool _isRemoteAudioPlaying = false; - private object _isRemoteAudioPlayingLock = new object(); + private object _isRemoteMediaPlayingLock = new object(); + private RemoteAudioTrack _remoteAudioTrack = null; + private RemoteVideoTrack _remoteVideoTrack = null; /// /// The underlying object. @@ -259,6 +263,7 @@ public static string GetDeviceName() public MainPage() { this.InitializeComponent(); + muteLocalVideoStroke.Visibility = Visibility.Collapsed; muteLocalAudioStroke.Visibility = Visibility.Collapsed; @@ -288,11 +293,10 @@ public MainPage() _peerConnection.IceStateChanged += OnIceStateChanged; _peerConnection.IceGatheringStateChanged += OnIceGatheringStateChanged; _peerConnection.RenegotiationNeeded += OnPeerRenegotiationNeeded; - _peerConnection.TrackAdded += Peer_RemoteTrackAdded; - _peerConnection.TrackRemoved += Peer_RemoteTrackRemoved; - _peerConnection.I420ARemoteVideoFrameReady += Peer_RemoteI420AFrameReady; - _peerConnection.LocalAudioFrameReady += Peer_LocalAudioFrameReady; - _peerConnection.RemoteAudioFrameReady += Peer_RemoteAudioFrameReady; + _peerConnection.AudioTrackAdded += Peer_RemoteAudioTrackAdded; + _peerConnection.AudioTrackRemoved += Peer_RemoteAudioTrackRemoved; + _peerConnection.VideoTrackAdded += Peer_RemoteVideoTrackAdded; + _peerConnection.VideoTrackRemoved += Peer_RemoteVideoTrackRemoved; //Window.Current.Closed += Shutdown; // doesn't work @@ -551,6 +555,10 @@ private async void OnLoaded(object sender, RoutedEventArgs e) { LogMessage("Initializing the WebRTC native plugin..."); + // Cannot run in UI thread on UWP because this will initialize the global factory + // (first library call) which needs to be done on a background thread. + await Task.Run(() => Library.ShutdownOptions = Library.ShutdownOptionsFlags.LogLiveObjects); + // Populate the combo box with the VideoProfileKind enum { var values = Enum.GetValues(typeof(VideoProfileKind)); @@ -630,8 +638,13 @@ private async void OnLoaded(object sender, RoutedEventArgs e) config.SdpSemantic = (sdpSemanticUnifiedPlan.IsChecked.GetValueOrDefault(true) ? SdpSemantic.UnifiedPlan : SdpSemantic.PlanB); await _peerConnection.InitializeAsync(config); + + // Add one audio and one video transceiver to signal the connection that it needs + // to negotiate one audio transport and one video transport with the remote peer. + _audioTransceiver = _peerConnection.AddTransceiver(MediaKind.Audio); + _videoTransceiver = _peerConnection.AddTransceiver(MediaKind.Video); } - catch(Exception ex) + catch (Exception ex) { LogMessage($"WebRTC native plugin init failed: {ex.Message}"); return; @@ -1120,7 +1133,7 @@ private void OnMediaEnded(MediaPlayer sender, object args) { //< TODO - This should never happen. But what to do with // local channels if it happens? - lock (_isLocalVideoPlayingLock) + lock (_isLocalMediaPlayingLock) { _isLocalVideoPlaying = false; } @@ -1135,7 +1148,7 @@ private void OnMediaEnded(MediaPlayer sender, object args) /// private async void StopLocalMedia() { - lock (_isLocalVideoPlayingLock) + lock (_isLocalMediaPlayingLock) { if (_isLocalVideoPlaying) { @@ -1147,6 +1160,7 @@ private async void StopLocalMedia() localMediaSource.Dispose(); localMediaSource = null; _isLocalVideoPlaying = false; + _localAudioTrack.AudioFrameReady -= LocalAudioTrack_FrameReady; _localVideoTrack.I420AVideoFrameReady -= LocalVideoTrack_I420AFrameReady; } } @@ -1155,15 +1169,17 @@ private async void StopLocalMedia() // signaling thread (and will block the caller thread), and audio processing will // delegate to the UI thread for UWP operations (and will block the signaling thread). await RunOnWorkerThread(() => { - lock (_isLocalVideoPlayingLock) + lock (_isLocalMediaPlayingLock) { - _peerConnection.RemoveLocalAudioTrack(); - _peerConnection.RemoveLocalVideoTrack(_localVideoTrack); // TODO - this doesn't unregister the callbacks... + _audioTransceiver.LocalAudioTrack = null; + _videoTransceiver.LocalVideoTrack = null; _renegotiationOfferEnabled = true; if (_peerConnection.IsConnected) { _peerConnection.CreateOffer(); } + _localAudioTrack.Dispose(); + _localAudioTrack = null; _localVideoTrack.Dispose(); _localVideoTrack = null; } @@ -1177,53 +1193,123 @@ await RunOnWorkerThread(() => { /// Currently does nothing, as starting the media pipeline is done lazily in the /// per-frame callback. /// - /// The kind of media track added (audio or video only). - /// - private void Peer_RemoteTrackAdded(PeerConnection.TrackKind trackKind) + /// The audio track added. + /// + private void Peer_RemoteAudioTrackAdded(RemoteAudioTrack track) { - LogMessage($"Added remote {trackKind} track."); - RunOnMainThread(() => + LogMessage($"Added remote audio track {track.Name}."); + lock (_isRemoteMediaPlayingLock) { - remoteVideoStatsTimer.Interval = TimeSpan.FromSeconds(1.0); - remoteVideoStatsTimer.Start(); - }); + if ((_remoteAudioTrack == null) && !_isRemoteAudioPlaying) + { + _remoteAudioTrack = track; + _remoteAudioTrack.AudioFrameReady += RemoteAudioTrack_FrameReady; + + // _isRemoteAudioPlaying will change to true once the first frame is received + } + } + } + + /// + /// Callback on remote media (audio or video) track added. + /// Currently does nothing, as starting the media pipeline is done lazily in the + /// per-frame callback. + /// + /// The video track added. + /// + private void Peer_RemoteVideoTrackAdded(RemoteVideoTrack track) + { + LogMessage($"Added remote video track {track.Name}."); + lock (_isRemoteMediaPlayingLock) + { + if ((_remoteVideoTrack == null) && !_isRemoteVideoPlaying) + { + _remoteVideoTrack = track; + _remoteVideoTrack.I420AVideoFrameReady += RemoteVideoTrack_I420AFrameReady; + + // _isRemoteVideoPlaying will change to true once the first frame is received + + RunOnMainThread(() => { + remoteVideoStatsTimer.Interval = TimeSpan.FromSeconds(1.0); + remoteVideoStatsTimer.Start(); + }); + } + } } /// /// Callback on remote media (audio or video) track removed. /// - /// The kind of media track added (audio or video only). - private void Peer_RemoteTrackRemoved(PeerConnection.TrackKind trackKind) + /// The audio track removed. + private void Peer_RemoteAudioTrackRemoved(Transceiver transceiver, RemoteAudioTrack track) { - LogMessage($"Removed remote {trackKind} track."); + LogMessage($"Removed remote audio track {track.Name} from transceiver {transceiver.Name}."); - if (trackKind == PeerConnection.TrackKind.Video) + // Just double-check that the remote audio track is indeed playing + // before scheduling a task to stop it. Currently the remote audio + // playback is exclusively controlled by the remote track being present + // or not, so these should be always in sync. + lock (_isRemoteMediaPlayingLock) { - // Just double-check that the remote video track is indeed playing - // before scheduling a task to stop it. Currently the remote video - // playback is exclusively controlled by the remote track being present - // or not, so these should be always in sync. - lock (_isRemoteVideoPlayingLock) + if ((_remoteAudioTrack == track) && _isRemoteAudioPlaying) { - if (_isRemoteVideoPlaying) - { - // Schedule on the main UI thread to access STA objects. - RunOnMainThread(() => { - remoteVideoStatsTimer.Stop(); - // Check that the remote video is still playing. - // This ensures that rapid calls to add/remove the video track - // are serialized, and an earlier remove call doesn't remove the - // track added by a later call, due to delays in scheduling the task. - lock (_isRemoteVideoPlayingLock) + _remoteAudioTrack.AudioFrameReady -= RemoteAudioTrack_FrameReady; + _remoteAudioTrack = null; + + // Schedule on the main UI thread to access STA objects. + RunOnMainThread(() => { + ClearRemoteAudioStats(); + // Check that the remote audio is still playing. + // This ensures that rapid calls to add/remove the audio track + // are serialized, and an earlier remove call doesn't remove the + // track added by a later call, due to delays in scheduling the task. + lock (_isRemoteMediaPlayingLock) + { + if (_isRemoteAudioPlaying) { - if (_isRemoteVideoPlaying) - { - remoteVideo.MediaPlayer.Pause(); - _isRemoteVideoPlaying = false; - } + _isRemoteAudioPlaying = false; } - }); - } + } + }); + } + } + } + + /// + /// Callback on remote media (audio or video) track removed. + /// + /// The video track removed. + private void Peer_RemoteVideoTrackRemoved(Transceiver transceiver, RemoteVideoTrack track) + { + LogMessage($"Removed remote video track {track.Name} from transceiver {transceiver.Name}."); + + // Just double-check that the remote video track is indeed playing + // before scheduling a task to stop it. Currently the remote video + // playback is exclusively controlled by the remote track being present + // or not, so these should be always in sync. + lock (_isRemoteMediaPlayingLock) + { + if ((_remoteVideoTrack == track) && _isRemoteVideoPlaying) + { + _remoteVideoTrack.I420AVideoFrameReady -= RemoteVideoTrack_I420AFrameReady; + _remoteVideoTrack = null; + + // Schedule on the main UI thread to access STA objects. + RunOnMainThread(() => { + remoteVideoStatsTimer.Stop(); + // Check that the remote video is still playing. + // This ensures that rapid calls to add/remove the video track + // are serialized, and an earlier remove call doesn't remove the + // track added by a later call, due to delays in scheduling the task. + lock (_isRemoteMediaPlayingLock) + { + if (_isRemoteVideoPlaying) + { + remoteVideo.MediaPlayer.Pause(); + _isRemoteVideoPlaying = false; + } + } + }); } } } @@ -1243,7 +1329,7 @@ private void LocalVideoTrack_I420AFrameReady(I420AVideoFrame frame) /// (or any other use). /// /// The newly received video frame. - private void Peer_RemoteI420AFrameReady(I420AVideoFrame frame) + private void RemoteVideoTrack_I420AFrameReady(I420AVideoFrame frame) { // Lazily start the remote video media player when receiving // the first video frame from the remote peer. Currently there @@ -1254,7 +1340,7 @@ private void Peer_RemoteI420AFrameReady(I420AVideoFrame frame) bool needNewSource = false; uint width = frame.width; uint height = frame.height; - lock (_isRemoteVideoPlayingLock) + lock (_isRemoteMediaPlayingLock) { if (!_isRemoteVideoPlaying) { @@ -1289,17 +1375,11 @@ private void Peer_RemoteI420AFrameReady(I420AVideoFrame frame) } /// - /// Callback on audio frame received from the local audio capture device (microphone), - /// for local output before (or in parallel of) being sent to the remote peer. + /// Callback on audio frame produced by the local peer. /// - /// The newly captured audio frame. - /// This is currently never called due to implementation limitations. - private void Peer_LocalAudioFrameReady(AudioFrame frame) + /// The newly produced audio frame. + private void LocalAudioTrack_FrameReady(AudioFrame frame) { - // The current underlying WebRTC implementation does not support - // local audio frame callbacks, so THIS WILL NEVER BE CALLED until - // that implementation is changed. - throw new NotImplementedException(); } /// @@ -1307,9 +1387,9 @@ private void Peer_LocalAudioFrameReady(AudioFrame frame) /// (or any other use). /// /// The newly received audio frame. - private void Peer_RemoteAudioFrameReady(AudioFrame frame) + private void RemoteAudioTrack_FrameReady(AudioFrame frame) { - lock (_isRemoteAudioPlayingLock) + lock (_isRemoteMediaPlayingLock) { uint channelCount = frame.channelCount; uint sampleRate = frame.sampleRate; @@ -1327,6 +1407,12 @@ private void Peer_RemoteAudioFrameReady(AudioFrame frame) } } + private void ClearRemoteAudioStats() + { + remoteAudioChannelCount.Text = "-"; + remoteAudioSampleRate.Text = "-"; + } + private void UpdateRemoteAudioStats(uint channelCount, uint sampleRate) { remoteAudioChannelCount.Text = channelCount.ToString(); @@ -1349,14 +1435,14 @@ private void MuteLocalVideoClicked(object sender, RoutedEventArgs e) private void MuteLocalAudioClicked(object sender, RoutedEventArgs e) { - if (_peerConnection.IsLocalAudioTrackEnabled()) + if (_localAudioTrack.Enabled) { - _peerConnection.SetLocalAudioTrackEnabled(false); + _localAudioTrack.Enabled = false; muteLocalAudioStroke.Visibility = Visibility.Visible; } else { - _peerConnection.SetLocalAudioTrackEnabled(true); + _localAudioTrack.Enabled = true; muteLocalAudioStroke.Visibility = Visibility.Collapsed; } } @@ -1452,11 +1538,12 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e) try { - // Add the local audio track captured from the local microphone - await _peerConnection.AddLocalAudioTrackAsync(); + // Create the local audio track captured from the local microphone + _localAudioTrack = await LocalAudioTrack.CreateFromDeviceAsync(); + _localAudioTrack.AudioFrameReady += LocalAudioTrack_FrameReady; - // Add the local video track captured from the local webcam - var trackConfig = new LocalVideoTrackSettings + // Create the local video track captured from the local webcam + var videoConfig = new LocalVideoTrackSettings { trackName = "local_video", videoDevice = captureDeviceInfo, @@ -1467,8 +1554,12 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e) framerate = framerate, enableMrc = false // TestAppUWP is a shared app, MRC will not get permission anyway }; - _localVideoTrack = await _peerConnection.AddLocalVideoTrackAsync(trackConfig); + _localVideoTrack = await LocalVideoTrack.CreateFromDeviceAsync(videoConfig); _localVideoTrack.I420AVideoFrameReady += LocalVideoTrack_I420AFrameReady; + + // Add the local tracks to the peer connection + _audioTransceiver.LocalAudioTrack = _localAudioTrack; + _videoTransceiver.LocalVideoTrack = _localVideoTrack; } catch (Exception ex) { @@ -1486,7 +1577,7 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e) muteLocalVideoStroke.Visibility = Visibility.Collapsed; localVideoSourceName.Text = $"({SelectedVideoCaptureDevice?.DisplayName})"; - lock (_isLocalVideoPlayingLock) + lock (_isLocalMediaPlayingLock) { _isLocalVideoPlaying = true; } diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp index ae00b4260..b46501f99 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp @@ -951,7 +951,8 @@ void PeerConnectionImpl::RemoveAllDataChannels() noexcept { // Invoke the DataChannelRemoved callback on the wrapper if any if (removed_cb) { - mrsDataChannelHandle data_native_handle = (void*)&data_channel; + DataChannel* const dc = data_channel.get(); + mrsDataChannelHandle data_native_handle = (void*)dc; removed_cb(data_native_handle); } @@ -1121,18 +1122,18 @@ void PeerConnectionImpl::Close() noexcept { for (auto&& transceiver : transceivers_) { if (auto remote_track = transceiver->GetRemoteTrack()) { if (remote_track->GetKind() == mrsTrackKind::kAudioTrack) { - auto* const audio_track = - static_cast(remote_track); + RefPtr audio_track( + static_cast(remote_track)); // keep alive audio_track->OnTrackRemoved(*this); if (audio_cb) { - audio_cb(audio_track, transceiver.get()); + audio_cb(audio_track.get(), transceiver.get()); } } else if (remote_track->GetKind() == mrsTrackKind::kVideoTrack) { - auto* const video_track = - static_cast(remote_track); + RefPtr video_track( + static_cast(remote_track)); // keep alive video_track->OnTrackRemoved(*this); if (video_cb) { - video_cb(video_track, transceiver.get()); + video_cb(video_track.get(), transceiver.get()); } } } @@ -1628,8 +1629,8 @@ PeerConnectionImpl::GetOrCreateTransceiverForNewRemoteTrack( int mline_index = -1; std::string name; std::vector stream_ids; - if (!ExtractTransceiverInfoFromReceiverPlanB(receiver, mline_index, - name, stream_ids)) { + if (!ExtractTransceiverInfoFromReceiverPlanB(receiver, mline_index, name, + stream_ids)) { return Error( Result::kUnknownError, "Failed to associate RTP receiver with Plan B emulated mline index " diff --git a/libs/Microsoft.MixedReality.WebRTC/DataChannel.cs b/libs/Microsoft.MixedReality.WebRTC/DataChannel.cs index ee2f1c9aa..96c231b6c 100644 --- a/libs/Microsoft.MixedReality.WebRTC/DataChannel.cs +++ b/libs/Microsoft.MixedReality.WebRTC/DataChannel.cs @@ -2,7 +2,7 @@ // Licensed under the MIT License. using System; -using System.Runtime.InteropServices; +using System.Diagnostics; using Microsoft.MixedReality.WebRTC.Interop; using Microsoft.MixedReality.WebRTC.Tracing; @@ -10,28 +10,34 @@ namespace Microsoft.MixedReality.WebRTC { /// /// Encapsulates a data channel of a peer connection. - /// - /// A data channel is a pipe allowing to send and receive arbitrary data to the + /// + /// A data channel is a "pipe" allowing to send and receive arbitrary data to the /// remote peer. Data channels are based on DTLS-SRTP, and are therefore secure (encrypted). /// Exact security guarantees are provided by the underlying WebRTC core implementation /// and the WebRTC standard itself. - /// + /// /// https://tools.ietf.org/wg/rtcweb/ - /// - /// An instance of is created by calling - /// or one of its variants. cannot be instantiated directly. + /// https://www.w3.org/TR/webrtc/ + /// + /// An instance of is created either by manually calling + /// or one of its variants, + /// or automatically by the implementation when a new data channel is created in-band by the + /// remote peer (). + /// cannot be instantiated directly. /// - public class DataChannel : IDisposable + /// + /// + /// + public class DataChannel { /// - /// Connecting state of a data channel, when adding it to a peer connection - /// or removing it from a peer connection. + /// Connection state of a data channel. /// public enum ChannelState { /// /// The data channel has just been created, and negotiating is underway to establish - /// a track between the peers. + /// a link between the peers. The data channel cannot be used to send/receive yet. /// Connecting = 0, @@ -60,13 +66,19 @@ public enum ChannelState /// Maximum buffering size, in bytes. public delegate void BufferingChangedDelegate(ulong previous, ulong current, ulong limit); - /// The object this data channel was created from. + /// + /// The object this data channel was created from and is attached to. + /// public PeerConnection PeerConnection { get; } - /// The unique identifier of the data channel in the current connection. + /// + /// The unique identifier of the data channel in the current connection. + /// public int ID { get; } - /// The data channel name in the current connection. + /// + /// The data channel name in the current connection. + /// public string Label { get; } /// @@ -89,22 +101,23 @@ public enum ChannelState public bool Reliable { get; } /// - /// The channel connection state represents the connection status when creating or closing the - /// data channel. Changes to this state are notified via the event. + /// The channel connection state represents the connection status. + /// Changes to this state are notified via the event. /// /// The channel connection state. /// - public ChannelState State { get; private set; } + public ChannelState State { get; internal set; } /// - /// Event fired when the data channel state changes, as reported by . + /// Event triggered when the data channel state changes. + /// The new state is available in . /// /// public event Action StateChanged; /// - /// Event fired when the data channel buffering changes. Monitor this to ensure calls to - /// do not fail. Internally the data channel contains + /// Event triggered when the data channel buffering changes. Users should monitor this to ensure + /// calls to do not fail. Internally the data channel contains /// a buffer of messages to send that could not be sent immediately, for example due to /// congestion control. Once this buffer is full, any further call to /// will fail until some mesages are processed and removed to make space. @@ -113,26 +126,31 @@ public enum ChannelState public event BufferingChangedDelegate BufferingChanged; /// - /// Event fires when a message is received through the data channel. + /// Event triggered when a message is received through the data channel. /// /// public event Action MessageReceived; /// - /// GC handle keeping the internal delegates alive while they are registered + /// Reference (GC handle) keeping the internal delegates alive while they are registered /// as callbacks with the native code. /// - private GCHandle _handle; + /// + private IntPtr _argsRef; /// - /// Handle to the native object this wrapper is associated with. + /// Handle to the native DataChannel object. /// - internal IntPtr _interopHandle = IntPtr.Zero; + /// + /// In native land this is a mrsDataChannelHandle. + /// + internal IntPtr _nativeHandle = IntPtr.Zero; - internal DataChannel(PeerConnection peer, GCHandle handle, - int id, string label, bool ordered, bool reliable) + internal DataChannel(IntPtr nativeHandle, PeerConnection peer, IntPtr argsRef, int id, string label, bool ordered, bool reliable) { - _handle = handle; + Debug.Assert(nativeHandle != IntPtr.Zero); + _nativeHandle = nativeHandle; + _argsRef = argsRef; PeerConnection = peer; ID = id; Label = label; @@ -142,25 +160,14 @@ internal DataChannel(PeerConnection peer, GCHandle handle, } /// - /// Finalizer to ensure the data track is removed from the peer connection - /// and the managed resources are cleaned-up. + /// Dispose of the native data channel. Invoked by its owner (). /// - ~DataChannel() + internal void DestroyNative() { - Dispose(); - } - - /// - /// Remove the data track from the peer connection and destroy it. - /// - public void Dispose() - { - State = ChannelState.Closing; - PeerConnection.RemoveDataChannel(_interopHandle); - _interopHandle = IntPtr.Zero; + _nativeHandle = IntPtr.Zero; State = ChannelState.Closed; - _handle.Free(); - GC.SuppressFinalize(this); + Utils.ReleaseWrapperRef(_argsRef); + _argsRef = IntPtr.Zero; } /// @@ -169,15 +176,21 @@ public void Dispose() /// The internal buffering is monitored via the event. /// /// The message to send to the remote peer. - /// The native data channel is not initialized. - /// The internal buffer is full. + /// The native data channel is not initialized. + /// The internal buffer is full. + /// The data channel is not open yet. /// /// /// public void SendMessage(byte[] message) { MainEventSource.Log.DataChannelSendMessage(ID, message.Length); - uint res = DataChannelInterop.DataChannel_SendMessage(_interopHandle, message, (ulong)message.LongLength); + // Check channel state before doing a P/Invoke call which would anyway fail + if (State != ChannelState.Open) + { + throw new DataChannelNotOpenException(); + } + uint res = DataChannelInterop.DataChannel_SendMessage(_nativeHandle, message, (ulong)message.LongLength); Utils.ThrowOnErrorCode(res); } diff --git a/libs/Microsoft.MixedReality.WebRTC/Exceptions.cs b/libs/Microsoft.MixedReality.WebRTC/Exceptions.cs index e5a234924..7d7049cba 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Exceptions.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Exceptions.cs @@ -58,4 +58,32 @@ public InvalidInteropNativeHandleException(string message, Exception inner) { } } + + /// + /// Exception thrown when trying to use a data channel that is not open. + /// + /// The user should listen to the event until the + /// property is + /// before trying to send some message with . + /// + public class DataChannelNotOpenException : Exception + { + /// + public DataChannelNotOpenException() + : base("Data channel is not open yet, cannot send message. Wait for the StateChanged event until State is Open.") + { + } + + /// + public DataChannelNotOpenException(string message) + : base(message) + { + } + + /// + public DataChannelNotOpenException(string message, Exception inner) + : base(message, inner) + { + } + } } diff --git a/libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs b/libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs index c10970aa8..fffc581eb 100644 --- a/libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs +++ b/libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.Diagnostics; using Microsoft.MixedReality.WebRTC.Interop; @@ -81,10 +82,14 @@ public void CompleteRequest(in Argb32VideoFrame frame) public class ExternalVideoTrackSource : IDisposable { /// - /// Once the external video track source is attached to some video track(s), this returns the peer connection - /// the video track(s) are part of. Otherwise this returns null. + /// A name for the external video track source, used for logging and debugging. /// - public PeerConnection PeerConnection { get; private set; } = null; + public string Name { get; set; } = string.Empty; + + /// + /// List of local video tracks this source is providing raw video frames to. + /// + public List Tracks { get; } = new List(); /// /// Handle to the native ExternalVideoTrackSource object. @@ -164,10 +169,6 @@ public void Dispose() return; } - // Remove the tracks associated with this source from the peer connection, if any - PeerConnection?.RemoveLocalVideoTracksFromSource(this); - Debug.Assert(PeerConnection == null); // see OnTracksRemovedFromSource - // Unregister and release the track callbacks ExternalVideoTrackSourceInterop.ExternalVideoTrackSource_Shutdown(_nativeHandle); Utils.ReleaseWrapperRef(_frameRequestCallbackArgsHandle); @@ -177,22 +178,43 @@ public void Dispose() _nativeHandle.Dispose(); } - internal void OnTracksAddedToSource(PeerConnection newConnection) + internal void OnTrackAddedToSource(LocalVideoTrack track) + { + Debug.Assert(!_nativeHandle.IsClosed); + Debug.Assert(!Tracks.Contains(track)); + Tracks.Add(track); + } + + internal void OnTrackRemovedFromSource(LocalVideoTrack track) { - Debug.Assert(PeerConnection == null); Debug.Assert(!_nativeHandle.IsClosed); - PeerConnection = newConnection; - var args = Utils.ToWrapper(_frameRequestCallbackArgsHandle); - args.Peer = newConnection; + bool removed = Tracks.Remove(track); + Debug.Assert(removed); } - internal void OnTracksRemovedFromSource(PeerConnection previousConnection) + internal void OnTracksRemovedFromSource(List tracks) { - Debug.Assert(PeerConnection == previousConnection); Debug.Assert(!_nativeHandle.IsClosed); - var args = Utils.ToWrapper(_frameRequestCallbackArgsHandle); - args.Peer = null; - PeerConnection = null; + var remainingTracks = new List(); + foreach (var track in tracks) + { + if (track.Source == this) + { + bool removed = Tracks.Remove(track); + Debug.Assert(removed); + } + else + { + remainingTracks.Add(track); + } + } + tracks = remainingTracks; + } + + /// + public override string ToString() + { + return $"(ExternalVideoTrackSource)\"{Name}\""; } } } diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs new file mode 100644 index 000000000..184140b59 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + internal class AudioTrackInterop + { + #region Native callbacks + + // The callbacks below ideally would use 'in', but that generates an error with .NET Native: + // "error : ILT0021: Could not resolve method 'EETypeRva:0x--------'". + // So instead use 'ref' to ensure the signature is compatible with the C++ const& signature. + + [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] + public delegate void AudioFrameUnmanagedCallback(IntPtr userData, ref AudioFrame frame); + + #endregion + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/DataChannelInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/DataChannelInterop.cs index f7bdd116d..405f5092e 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/DataChannelInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/DataChannelInterop.cs @@ -11,6 +11,18 @@ internal class DataChannelInterop { #region Native functions + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsDataChannelSetUserData")] + public static unsafe extern void DataChannel_SetUserData(IntPtr handle, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsDataChannelGetUserData")] + public static unsafe extern IntPtr DataChannel_GetUserData(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsDataChannelRegisterCallbacks")] + public static extern void DataChannel_RegisterCallbacks(IntPtr dataChannelHandle, ref Callbacks callbacks); + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsDataChannelSendMessage")] public static extern uint DataChannel_SendMessage(IntPtr dataChannelHandle, byte[] data, ulong size); @@ -24,8 +36,8 @@ internal class DataChannelInterop public ref struct CreateConfig { public int id; - public string label; public uint flags; + public string label; } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -44,10 +56,6 @@ public ref struct Callbacks #region Native callbacks - [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] - public delegate IntPtr CreateObjectDelegate(IntPtr peer, CreateConfig config, - out Callbacks callbacks); - [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] public delegate void MessageCallback(IntPtr userData, IntPtr data, ulong size); @@ -71,15 +79,6 @@ public class CallbackArgs public StateCallback StateCallback; } - [MonoPInvokeCallback(typeof(CreateObjectDelegate))] - public static IntPtr DataChannelCreateObjectCallback(IntPtr peer, CreateConfig config, - out Callbacks callbacks) - { - var peerWrapper = Utils.ToWrapper(peer); - var dataChannelWrapper = CreateWrapper(peerWrapper, config, out callbacks); - return Utils.MakeWrapperRef(dataChannelWrapper); - } - [MonoPInvokeCallback(typeof(MessageCallback))] public static void DataChannelMessageCallback(IntPtr userData, IntPtr data, ulong size) { @@ -106,7 +105,7 @@ public static void DataChannelStateCallback(IntPtr userData, int state, int id) #region Utilities - public static DataChannel CreateWrapper(PeerConnection parent, CreateConfig config, out Callbacks callbacks) + public static DataChannel CreateWrapper(PeerConnection parent, in PeerConnectionInterop.DataChannelAddedInfo info) { // Create the callback args for the data channel var args = new CallbackArgs() @@ -119,37 +118,33 @@ public static DataChannel CreateWrapper(PeerConnection parent, CreateConfig conf }; // Pin the args to pin the delegates while they're registered with the native code - var handle = GCHandle.Alloc(args, GCHandleType.Normal); - IntPtr userData = GCHandle.ToIntPtr(handle); + IntPtr argsRef = Utils.MakeWrapperRef(args); - // Create a new data channel. It will hold the lock for its args while alive. - bool ordered = (config.flags & 0x1) != 0; - bool reliable = (config.flags & 0x2) != 0; - var dataChannel = new DataChannel(parent, handle, config.id, config.label, ordered, reliable); + // Create a new data channel wrapper. It will hold the lock for its args while alive. + bool ordered = (info.flags & 0x1) != 0; + bool reliable = (info.flags & 0x2) != 0; + var dataChannel = new DataChannel(info.dataChannelHandle, parent, argsRef, info.id, info.label, ordered, reliable); args.DataChannel = dataChannel; - // Fill out the callbacks - callbacks = new Callbacks() + // Assign a reference to it inside the UserData of the native object so it can be retrieved whenever needed + IntPtr wrapperRef = Utils.MakeWrapperRef(dataChannel); + DataChannel_SetUserData(info.dataChannelHandle, wrapperRef); + + // Register the callbacks + var callbacks = new Callbacks() { messageCallback = args.MessageCallback, - messageUserData = userData, + messageUserData = argsRef, bufferingCallback = args.BufferingCallback, - bufferingUserData = userData, + bufferingUserData = argsRef, stateCallback = args.StateCallback, - stateUserData = userData + stateUserData = argsRef }; + DataChannel_RegisterCallbacks(info.dataChannelHandle, ref callbacks); return dataChannel; } - public static void SetHandle(DataChannel dataChannel, IntPtr handle) - { - Debug.Assert(handle != IntPtr.Zero); - // Either first-time assign or no-op (assign same value again) - Debug.Assert((dataChannel._interopHandle == IntPtr.Zero) || (dataChannel._interopHandle == handle)); - dataChannel._interopHandle = handle; - } - #endregion } } diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/ExternalVideoTrackSourceInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/ExternalVideoTrackSourceInterop.cs index bebb9acd6..7fcb86da2 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/ExternalVideoTrackSourceInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/ExternalVideoTrackSourceInterop.cs @@ -100,7 +100,6 @@ public static void RequestArgb32VideoFrameFromExternalSourceCallback(IntPtr user public class VideoFrameRequestCallbackArgs { - public PeerConnection Peer; public ExternalVideoTrackSource Source; } @@ -166,7 +165,6 @@ public static ExternalVideoTrackSource CreateExternalVideoTrackSourceFromI420ACa // Create some static callback args which keep the sourceDelegate alive var args = new I420AVideoFrameRequestCallbackArgs { - Peer = null, // set when track added Source = null, // set below FrameRequestCallback = frameRequestCallback, // This wraps the method into a temporary System.Delegate object, which is then assigned to @@ -206,7 +204,6 @@ public static ExternalVideoTrackSource CreateExternalVideoTrackSourceFromArgb32C // Create some static callback args which keep the sourceDelegate alive var args = new Argb32VideoFrameRequestCallbackArgs { - Peer = null, // set when track added Source = null, // set below FrameRequestCallback = frameRequestCallback, // This wraps the method into a temporary System.Delegate object, which is then assigned to diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs index 4cb730b1a..9f92a02ce 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/InteropUtils.cs @@ -3,6 +3,7 @@ using Microsoft.MixedReality.WebRTC.Tracing; using System; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Text; @@ -60,16 +61,16 @@ internal static class Utils internal const uint MRS_E_SCTP_NOT_NEGOTIATED = 0x80000301u; internal const uint MRS_E_INVALID_DATA_CHANNEL_ID = 0x80000302u; - public static IntPtr MakeWrapperRef(object obj) + public static IntPtr MakeWrapperRef(object wrapper) { - var handle = GCHandle.Alloc(obj, GCHandleType.Normal); - var arg = GCHandle.ToIntPtr(handle); - return arg; + var handle = GCHandle.Alloc(wrapper, GCHandleType.Normal); + var wrapperRef = GCHandle.ToIntPtr(handle); + return wrapperRef; } - public static T ToWrapper(IntPtr peer) where T : class + public static T ToWrapper(IntPtr wrapperRef) where T : class { - var handle = GCHandle.FromIntPtr(peer); + var handle = GCHandle.FromIntPtr(wrapperRef); var wrapper = (handle.Target as T); return wrapper; } @@ -209,5 +210,14 @@ public static void ThrowOnErrorCode(uint res) /// [DllImport(dllPath, CallingConvention = CallingConvention.StdCall, EntryPoint = "mrsSetFrameHeightRoundMode")] public static unsafe extern void SetFrameHeightRoundMode(PeerConnection.FrameHeightRoundMode value); + + public static string EncodeTransceiverStreamIDs(List streamIDs) + { + if ((streamIDs == null) || (streamIDs.Count == 0)) + { + return string.Empty; + } + return string.Join(";", streamIDs.ToArray()); + } } } diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs new file mode 100644 index 000000000..4ab1d1e5d --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; +using static Microsoft.MixedReality.WebRTC.Interop.AudioTrackInterop; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + /// + /// Handle to a native local audio track object. + /// + public sealed class LocalAudioTrackHandle : SafeHandle + { + /// + /// Check if the current handle is invalid, which means it is not referencing + /// an actual native object. Note that a valid handle only means that the internal + /// handle references a native object, but does not guarantee that the native + /// object is still accessible. It is only safe to access the native object if + /// the handle is not closed, which implies it being valid. + /// + public override bool IsInvalid + { + get + { + return (handle == IntPtr.Zero); + } + } + + /// + /// Default constructor for an invalid handle. + /// + public LocalAudioTrackHandle() : base(IntPtr.Zero, ownsHandle: true) + { + } + + /// + /// Constructor for a valid handle referencing the given native object. + /// + /// The valid internal handle to the native object. + public LocalAudioTrackHandle(IntPtr handle) : base(IntPtr.Zero, ownsHandle: true) + { + SetHandle(handle); + } + + /// + /// Release the native object while the handle is being closed. + /// + /// Return true if the native object was successfully released. + protected override bool ReleaseHandle() + { + LocalAudioTrackInterop.LocalAudioTrack_RemoveRef(handle); + return true; + } + } + + internal class LocalAudioTrackInterop + { + #region Native functions + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackAddRef")] + public static unsafe extern void LocalAudioTrack_AddRef(LocalAudioTrackHandle handle); + + // Note - This is used during SafeHandle.ReleaseHandle(), so cannot use LocalAudioTrackHandle + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackRemoveRef")] + public static unsafe extern void LocalAudioTrack_RemoveRef(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackCreateFromDevice")] + public static unsafe extern uint LocalAudioTrack_CreateFromDevice(in PeerConnectionInterop.LocalAudioTrackInteropInitConfig config, + string trackName, out LocalAudioTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackRegisterFrameCallback")] + public static extern void LocalAudioTrack_RegisterFrameCallback(LocalAudioTrackHandle trackHandle, + AudioFrameUnmanagedCallback callback, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackIsEnabled")] + public static extern int LocalAudioTrack_IsEnabled(LocalAudioTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalAudioTrackSetEnabled")] + public static extern uint LocalAudioTrack_SetEnabled(LocalAudioTrackHandle trackHandle, int enabled); + + #endregion + + public class InteropCallbackArgs + { + public LocalAudioTrack Track; + public AudioFrameUnmanagedCallback FrameCallback; + } + + [MonoPInvokeCallback(typeof(AudioFrameUnmanagedCallback))] + public static void FrameCallback(IntPtr userData, ref AudioFrame frame) + { + var track = Utils.ToWrapper(userData); + track.OnFrameReady(frame); + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/LocalVideoTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalVideoTrackInterop.cs index 34319a755..01ecd0932 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/LocalVideoTrackInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalVideoTrackInterop.cs @@ -55,7 +55,9 @@ protected override bool ReleaseHandle() internal class LocalVideoTrackInterop { - #region Native callbacks + #region Unmanaged delegates + + // Note - none of those method arguments can be SafeHandle; use IntPtr instead. // The callbacks below ideally would use 'in', but that generates an error with .NET Native: // "error : ILT0021: Could not resolve method 'EETypeRva:0x--------'". @@ -81,6 +83,17 @@ internal class LocalVideoTrackInterop EntryPoint = "mrsLocalVideoTrackRemoveRef")] public static unsafe extern void LocalVideoTrack_RemoveRef(IntPtr handle); + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalVideoTrackCreateFromDevice")] + public static unsafe extern uint LocalVideoTrack_CreateFromDevice(in PeerConnectionInterop.LocalVideoTrackInteropInitConfig config, + string trackName, out LocalVideoTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsLocalVideoTrackCreateFromExternalSource")] + public static unsafe extern uint LocalVideoTrack_CreateFromExternalSource( + in PeerConnectionInterop.LocalVideoTrackFromExternalSourceInteropInitConfig config, + out LocalVideoTrackHandle trackHandle); + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsLocalVideoTrackRegisterI420AFrameCallback")] public static extern void LocalVideoTrack_RegisterI420AFrameCallback(LocalVideoTrackHandle trackHandle, diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs index f8996f758..372efc9b3 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs @@ -61,18 +61,11 @@ internal class PeerConnectionInterop { // Types of trampolines for MonoPInvokeCallback private delegate void ConnectedDelegate(IntPtr peer); - private delegate void DataChannelAddedDelegate(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle); - private delegate void DataChannelRemovedDelegate(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle); private delegate void LocalSdpReadytoSendDelegate(IntPtr peer, string type, string sdp); private delegate void IceCandidateReadytoSendDelegate(IntPtr peer, string candidate, int sdpMlineindex, string sdpMid); private delegate void IceStateChangedDelegate(IntPtr peer, IceConnectionState newState); private delegate void IceGatheringStateChangedDelegate(IntPtr peer, IceGatheringState newState); private delegate void RenegotiationNeededDelegate(IntPtr peer); - private delegate void TrackAddedDelegate(IntPtr peer, PeerConnection.TrackKind trackKind); - private delegate void TrackRemovedDelegate(IntPtr peer, PeerConnection.TrackKind trackKind); - private delegate void DataChannelMessageDelegate(IntPtr peer, IntPtr data, ulong size); - private delegate void DataChannelBufferingDelegate(IntPtr peer, ulong previous, ulong current, ulong limit); - private delegate void DataChannelStateDelegate(IntPtr peer, int state, int id); // Callbacks for internal enumeration implementation only public delegate void VideoCaptureDeviceEnumCallbackImpl(string id, string name); @@ -127,20 +120,6 @@ public static void VideoCaptureFormat_EnumCompletedCallback(uint resultCode, Int wrapper.completedCallback(exception); // this is optional, allows to be null } - /// - /// Utility to lock all low-level interop delegates registered with the native plugin for the duration - /// of the peer connection wrapper lifetime, and prevent their garbage collection. - /// - /// - /// The delegate don't need to be pinned, just referenced to prevent garbage collection. - /// So referencing them from this class is enough to keep them alive and usable. - /// - public class InteropCallbacks - { - public PeerConnection Peer; - public DataChannelInterop.CreateObjectDelegate DataChannelCreateObjectCallback; - } - /// /// Utility to lock all optional delegates registered with the native plugin for the duration /// of the peer connection wrapper lifetime, and prevent their garbage collection. @@ -160,14 +139,11 @@ public class PeerCallbackArgs public PeerConnectionIceStateChangedCallback IceStateChangedCallback; public PeerConnectionIceGatheringStateChangedCallback IceGatheringStateChangedCallback; public PeerConnectionRenegotiationNeededCallback RenegotiationNeededCallback; - public PeerConnectionTrackAddedCallback TrackAddedCallback; - public PeerConnectionTrackRemovedCallback TrackRemovedCallback; - public LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback I420ALocalVideoFrameCallback; - public LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback I420ARemoteVideoFrameCallback; - public LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback Argb32LocalVideoFrameCallback; - public LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback Argb32RemoteVideoFrameCallback; - public AudioFrameUnmanagedCallback LocalAudioFrameCallback; - public AudioFrameUnmanagedCallback RemoteAudioFrameCallback; + public PeerConnectionTransceiverAddedCallback TransceiverAddedCallback; + public PeerConnectionAudioTrackAddedCallback AudioTrackAddedCallback; + public PeerConnectionAudioTrackRemovedCallback AudioTrackRemovedCallback; + public PeerConnectionVideoTrackAddedCallback VideoTrackAddedCallback; + public PeerConnectionVideoTrackRemovedCallback VideoTrackRemovedCallback; } [MonoPInvokeCallback(typeof(ConnectedDelegate))] @@ -177,24 +153,24 @@ public static void ConnectedCallback(IntPtr userData) peer.OnConnected(); } - [MonoPInvokeCallback(typeof(DataChannelAddedDelegate))] - public static void DataChannelAddedCallback(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle) + [MonoPInvokeCallback(typeof(PeerConnectionDataChannelAddedCallback))] + public static void DataChannelAddedCallback(IntPtr userData, in DataChannelAddedInfo info) { - var peerWrapper = Utils.ToWrapper(peer); - var dataChannelWrapper = Utils.ToWrapper(dataChannel); - // Ensure that the DataChannel wrapper knows about its native object. - // This is not always the case, if created via the interop constructor, - // as the wrapper is created before the native object exists. - DataChannelInterop.SetHandle(dataChannelWrapper, dataChannelHandle); + var peerWrapper = Utils.ToWrapper(userData); + var dataChannelWrapper = DataChannelInterop.CreateWrapper(peerWrapper, in info); peerWrapper.OnDataChannelAdded(dataChannelWrapper); } - [MonoPInvokeCallback(typeof(DataChannelRemovedDelegate))] - public static void DataChannelRemovedCallback(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle) + [MonoPInvokeCallback(typeof(PeerConnectionDataChannelRemovedCallback))] + public static void DataChannelRemovedCallback(IntPtr userData, IntPtr dataChannelHandle) { - var peerWrapper = Utils.ToWrapper(peer); - var dataChannelWrapper = Utils.ToWrapper(dataChannel); + var peerWrapper = Utils.ToWrapper(userData); + IntPtr dataChannelRef = DataChannelInterop.DataChannel_GetUserData(dataChannelHandle); + DataChannelInterop.DataChannel_SetUserData(dataChannelHandle, IntPtr.Zero); + var dataChannelWrapper = Utils.ToWrapper(dataChannelRef); peerWrapper.OnDataChannelRemoved(dataChannelWrapper); + dataChannelWrapper.DestroyNative(); + Utils.ReleaseWrapperRef(dataChannelRef); } [MonoPInvokeCallback(typeof(LocalSdpReadytoSendDelegate))] @@ -232,46 +208,59 @@ public static void RenegotiationNeededCallback(IntPtr userData) peer.OnRenegotiationNeeded(); } - [MonoPInvokeCallback(typeof(TrackAddedDelegate))] - public static void TrackAddedCallback(IntPtr userData, PeerConnection.TrackKind trackKind) + [MonoPInvokeCallback(typeof(PeerConnectionTransceiverAddedCallback))] + public static void TransceiverAddedCallback(IntPtr peer, in TransceiverAddedInfo info) { - var peer = Utils.ToWrapper(userData); - peer.OnTrackAdded(trackKind); + var peerWrapper = Utils.ToWrapper(peer); + var transceiverWrapper = TransceiverInterop.CreateWrapper(peerWrapper, in info); + peerWrapper.OnTransceiverAdded(transceiverWrapper); } - [MonoPInvokeCallback(typeof(TrackRemovedDelegate))] - public static void TrackRemovedCallback(IntPtr userData, PeerConnection.TrackKind trackKind) + [MonoPInvokeCallback(typeof(PeerConnectionAudioTrackAddedCallback))] + public static void AudioTrackAddedCallback(IntPtr peer, in RemoteAudioTrackAddedInfo info) { - var peer = Utils.ToWrapper(userData); - peer.OnTrackRemoved(trackKind); + var peerWrapper = Utils.ToWrapper(peer); + IntPtr transceiver = TransceiverInterop.Transceiver_GetUserData(info.audioTransceiverHandle); + Debug.Assert(transceiver != IntPtr.Zero); // must have been set by the TransceiverAdded event + var transceiverWrapper = Utils.ToWrapper(transceiver); + var remoteAudioTrackWrapper = RemoteAudioTrackInterop.CreateWrapper(peerWrapper, in info); + peerWrapper.OnAudioTrackAdded(remoteAudioTrackWrapper, transceiverWrapper); } - [MonoPInvokeCallback(typeof(LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback))] - public static void I420ARemoteVideoFrameCallback(IntPtr userData, ref I420AVideoFrame frame) + [MonoPInvokeCallback(typeof(PeerConnectionAudioTrackRemovedCallback))] + public static void AudioTrackRemovedCallback(IntPtr userData, IntPtr audioTrackHandle, IntPtr audioTransceiverHandle) { - var peer = Utils.ToWrapper(userData); - peer.OnI420ARemoteVideoFrameReady(frame); + var peerWrapper = Utils.ToWrapper(userData); + IntPtr audioTrackRef = RemoteAudioTrackInterop.RemoteAudioTrack_GetUserData(audioTrackHandle); + var audioTrackWrapper = Utils.ToWrapper(audioTrackRef); + peerWrapper.OnAudioTrackRemoved(audioTrackWrapper); } - [MonoPInvokeCallback(typeof(LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback))] - public static void Argb32RemoteVideoFrameCallback(IntPtr userData, ref Argb32VideoFrame frame) + [MonoPInvokeCallback(typeof(PeerConnectionVideoTrackAddedCallback))] + public static void VideoTrackAddedCallback(IntPtr peer, in RemoteVideoTrackAddedInfo info) { - var peer = Utils.ToWrapper(userData); - peer.OnArgb32RemoteVideoFrameReady(frame); + var peerWrapper = Utils.ToWrapper(peer); + IntPtr transceiver = TransceiverInterop.Transceiver_GetUserData(info.videoTransceiverHandle); + Debug.Assert(transceiver != IntPtr.Zero); // must have been set by the TransceiverAdded event + var transceiverWrapper = Utils.ToWrapper(transceiver); + var remoteVideoTrackWrapper = RemoteVideoTrackInterop.CreateWrapper(peerWrapper, in info); + peerWrapper.OnVideoTrackAdded(remoteVideoTrackWrapper, transceiverWrapper); } - [MonoPInvokeCallback(typeof(AudioFrameUnmanagedCallback))] - public static void LocalAudioFrameCallback(IntPtr userData, ref AudioFrame frame) + [MonoPInvokeCallback(typeof(PeerConnectionVideoTrackRemovedCallback))] + public static void VideoTrackRemovedCallback(IntPtr userData, IntPtr videoTrackHandle, IntPtr videoTransceiverHandle) { - var peer = Utils.ToWrapper(userData); - peer.OnLocalAudioFrameReady(frame); + var peerWrapper = Utils.ToWrapper(userData); + IntPtr videoTrackRef = RemoteVideoTrackInterop.RemoteVideoTrack_GetUserData(videoTrackHandle); + var videoTrackWrapper = Utils.ToWrapper(videoTrackRef); + peerWrapper.OnVideoTrackRemoved(videoTrackWrapper); } - [MonoPInvokeCallback(typeof(AudioFrameUnmanagedCallback))] - public static void RemoteAudioFrameCallback(IntPtr userData, ref AudioFrame frame) + [MonoPInvokeCallback(typeof(ActionDelegate))] + public static void RemoteDescriptionApplied(IntPtr args) { - var peer = Utils.ToWrapper(userData); - peer.OnRemoteAudioFrameReady(frame); + var remoteDesc = Utils.ToWrapper(args); + remoteDesc.completedEvent.Set(); } public static readonly PeerConnectionSimpleStatsCallback SimpleStatsReportDelegate = SimpleStatsReportCallback; @@ -334,22 +323,10 @@ public static IEnumerable GetStatsObject(PeerConnection.StatsReport.Handle return res; } - [MonoPInvokeCallback(typeof(ActionDelegate))] - public static void RemoteDescriptionApplied(IntPtr args) - { - var remoteDesc = Utils.ToWrapper(args); - remoteDesc.completedEvent.Set(); - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct MarshaledInteropCallbacks - { - public DataChannelInterop.CreateObjectDelegate DataChannelCreateObjectCallback; - } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct PeerConnectionConfiguration + internal ref struct PeerConnectionConfiguration { + [MarshalAs(UnmanagedType.LPStr)] public string EncodedIceServers; public IceTransportType IceTransportType; public BundlePolicy BundlePolicy; @@ -357,17 +334,29 @@ internal struct PeerConnectionConfiguration } /// - /// Helper structure to pass parameters to the native implementation when creating a local video track - /// by opening a local video capture device. + /// Helper structure to pass parameters to the native implementation when creating a local audio track. /// [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal ref struct LocalVideoTrackInteropInitConfig + internal ref struct LocalAudioTrackInteropInitConfig { /// - /// Handle to the local video track wrapper. + /// Constructor for creating a local audio track. /// - public IntPtr trackHandle; + /// The newly created track wrapper. + /// The settings to initialize the newly created native track. + /// + public LocalAudioTrackInteropInitConfig(LocalAudioTrack track, LocalAudioTrackSettings settings) + { + } + } + /// + /// Helper structure to pass parameters to the native implementation when creating a local video track + /// by opening a local video capture device. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal ref struct LocalVideoTrackInteropInitConfig + { /// /// Video capture device unique identifier, as returned by . /// @@ -430,11 +419,9 @@ internal ref struct LocalVideoTrackInteropInitConfig /// /// The newly created track wrapper. /// The settings to initialize the newly created native track. - /// + /// public LocalVideoTrackInteropInitConfig(LocalVideoTrack track, LocalVideoTrackSettings settings) { - trackHandle = Utils.MakeWrapperRef(track); - if (settings != null) { VideoDeviceId = settings.videoDevice.id; @@ -468,23 +455,136 @@ public LocalVideoTrackInteropInitConfig(LocalVideoTrack track, LocalVideoTrackSe internal ref struct LocalVideoTrackFromExternalSourceInteropInitConfig { /// - /// Handle to the wrapper for the native local video track that will - /// be created. + /// Handle to native external video track source. + /// + public IntPtr SourceHandle; + + /// + /// Name of the newly-created track. This must be a valid SDP token. /// - public IntPtr LocalVideoTrackWrapperHandle; + [MarshalAs(UnmanagedType.LPStr)] + public string TrackName; /// /// Constructor for creating a local video track from a wrapper and an existing external source. /// - /// The newly created track wrapper. /// The external source to use with the newly created native track. - /// - public LocalVideoTrackFromExternalSourceInteropInitConfig(LocalVideoTrack track, ExternalVideoTrackSource source) + /// The newly created track name. This must be a valid SDP token. + /// + public LocalVideoTrackFromExternalSourceInteropInitConfig(string trackName, ExternalVideoTrackSource source) { - LocalVideoTrackWrapperHandle = Utils.MakeWrapperRef(track); + SourceHandle = source._nativeHandle.DangerousGetHandle(); + TrackName = trackName; } } + /// + /// Marshalling structure to receive information about a newly created data channel + /// just added to the peer connection after a remote description was applied. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal struct DataChannelAddedInfo + { + /// + /// Handle of the newly created data channel. + /// + public IntPtr dataChannelHandle; + + public int id; + public uint flags; + public string label; + } + + /// + /// Marshalling structure to receive information about a newly created transceiver + /// just added to the peer connection. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal struct TransceiverAddedInfo + { + /// + /// Handle to the newly-created native transceiver object. + /// + public IntPtr transceiverHandle; + + /// + /// Transceiver name. + /// + [MarshalAs(UnmanagedType.LPStr)] + public string name; + + /// + /// Kind of media the transceiver transports. + /// + public MediaKind mediaKind; + + /// + /// Media line index of the transceiver. + /// + public int mlineIndex; + + /// + /// Encoded string of semi-colon separated list of stream IDs. + /// Example for stream IDs ("id1", "id2", "id3"): + /// encodedStreamIDs = "id1;id2;id3"; + /// + [MarshalAs(UnmanagedType.LPStr)] + public string encodedStreamIDs; + + /// + /// Initial desired direction of the transceiver on creation. + /// + public Transceiver.Direction desiredDirection; + } + + /// + /// Marshalling structure to receive information about a newly created remote audio track + /// just added to a transceiver of the peer connection. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal struct RemoteAudioTrackAddedInfo + { + /// + /// Handle of the newly created remote audio track. + /// + public IntPtr trackHandle; + + /// + /// Handle of the audio transceiver the track was added to. + /// + public IntPtr audioTransceiverHandle; + + /// + /// Name of the remote audio track. + /// + [MarshalAs(UnmanagedType.LPStr)] + public string trackName; + } + + /// + /// Marshalling structure to receive information about a newly created remote video track + /// just added to a transceiver of the peer connection. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal struct RemoteVideoTrackAddedInfo + { + /// + /// Handle of the newly created remote video track. + /// + public IntPtr trackHandle; + + /// + /// Handle of the video transceiver the track was added to. + /// + public IntPtr videoTransceiverHandle; + + /// + /// Name of the remote video track. + /// + [MarshalAs(UnmanagedType.LPStr)] + public string trackName; + } + #region Reverse P/Invoke delegates @@ -503,10 +603,10 @@ public LocalVideoTrackFromExternalSourceInteropInitConfig(LocalVideoTrack track, public delegate void VideoCaptureFormatEnumCompletedCallback(uint resultCode, IntPtr userData); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] - public delegate void PeerConnectionDataChannelAddedCallback(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle); + public delegate void PeerConnectionDataChannelAddedCallback(IntPtr userData, in DataChannelAddedInfo info); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] - public delegate void PeerConnectionDataChannelRemovedCallback(IntPtr peer, IntPtr dataChannel, IntPtr dataChannelHandle); + public delegate void PeerConnectionDataChannelRemovedCallback(IntPtr userData, IntPtr dataChannelHandle); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] public delegate void PeerConnectionInteropCallbacks(IntPtr userData); @@ -534,10 +634,19 @@ public delegate void PeerConnectionIceGatheringStateChangedCallback(IntPtr userD public delegate void PeerConnectionRenegotiationNeededCallback(IntPtr userData); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] - public delegate void PeerConnectionTrackAddedCallback(IntPtr userData, PeerConnection.TrackKind trackKind); + public delegate void PeerConnectionTransceiverAddedCallback(IntPtr peerHandle, in TransceiverAddedInfo info); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] - public delegate void PeerConnectionTrackRemovedCallback(IntPtr userData, PeerConnection.TrackKind trackKind); + public delegate void PeerConnectionAudioTrackAddedCallback(IntPtr peerHandle, in RemoteAudioTrackAddedInfo info); + + [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] + public delegate void PeerConnectionAudioTrackRemovedCallback(IntPtr peerHandle, IntPtr audioTrackHandle, IntPtr audioTransceiverHandle); + + [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] + public delegate void PeerConnectionVideoTrackAddedCallback(IntPtr peerHandle, in RemoteVideoTrackAddedInfo info); + + [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] + public delegate void PeerConnectionVideoTrackRemovedCallback(IntPtr peerHandle, IntPtr videoTrackHandle, IntPtr videoTransceiverHandle); [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] public delegate void AudioFrameUnmanagedCallback(IntPtr userData, ref AudioFrame frame); @@ -551,6 +660,7 @@ public delegate void PeerConnectionIceGatheringStateChangedCallback(IntPtr userD [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] public delegate void ActionDelegate(IntPtr peer); + #endregion @@ -577,13 +687,7 @@ public static extern uint EnumVideoCaptureFormatsAsync(string deviceId, VideoCap [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsPeerConnectionCreate")] - public static extern uint PeerConnection_Create(PeerConnectionConfiguration config, IntPtr peer, - out PeerConnectionHandle peerHandleOut); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterInteropCallbacks")] - public static extern void PeerConnection_RegisterInteropCallbacks(PeerConnectionHandle peerHandle, - in MarshaledInteropCallbacks callback); + public static extern uint PeerConnection_Create(ref PeerConnectionConfiguration config, out PeerConnectionHandle peerHandleOut); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsPeerConnectionRegisterConnectedCallback")] @@ -616,92 +720,54 @@ public static extern void PeerConnection_RegisterRenegotiationNeededCallback(Pee PeerConnectionRenegotiationNeededCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterTrackAddedCallback")] - public static extern void PeerConnection_RegisterTrackAddedCallback(PeerConnectionHandle peerHandle, - PeerConnectionTrackAddedCallback callback, IntPtr userData); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterTrackRemovedCallback")] - public static extern void PeerConnection_RegisterTrackRemovedCallback(PeerConnectionHandle peerHandle, - PeerConnectionTrackRemovedCallback callback, IntPtr userData); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterDataChannelAddedCallback")] - public static extern void PeerConnection_RegisterDataChannelAddedCallback(PeerConnectionHandle peerHandle, - PeerConnectionDataChannelAddedCallback callback, IntPtr userData); + EntryPoint = "mrsPeerConnectionRegisterTransceiverAddedCallback")] + public static extern void PeerConnection_RegisterTransceiverAddedCallback(PeerConnectionHandle peerHandle, + PeerConnectionTransceiverAddedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterDataChannelRemovedCallback")] - public static extern void PeerConnection_RegisterDataChannelRemovedCallback(PeerConnectionHandle peerHandle, - PeerConnectionDataChannelRemovedCallback callback, IntPtr userData); + EntryPoint = "mrsPeerConnectionRegisterAudioTrackAddedCallback")] + public static extern void PeerConnection_RegisterAudioTrackAddedCallback(PeerConnectionHandle peerHandle, + PeerConnectionAudioTrackAddedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterI420ARemoteVideoFrameCallback")] - public static extern void PeerConnection_RegisterI420ARemoteVideoFrameCallback(PeerConnectionHandle peerHandle, - LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback callback, IntPtr userData); + EntryPoint = "mrsPeerConnectionRegisterAudioTrackRemovedCallback")] + public static extern void PeerConnection_RegisterAudioTrackRemovedCallback(PeerConnectionHandle peerHandle, + PeerConnectionAudioTrackRemovedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterArgb32RemoteVideoFrameCallback")] - public static extern void PeerConnection_RegisterArgb32RemoteVideoFrameCallback(PeerConnectionHandle peerHandle, - LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback callback, IntPtr userData); + EntryPoint = "mrsPeerConnectionRegisterVideoTrackAddedCallback")] + public static extern void PeerConnection_RegisterVideoTrackAddedCallback(PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackAddedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterLocalAudioFrameCallback")] - public static extern void PeerConnection_RegisterLocalAudioFrameCallback(PeerConnectionHandle peerHandle, - AudioFrameUnmanagedCallback callback, IntPtr userData); + EntryPoint = "mrsPeerConnectionRegisterVideoTrackRemovedCallback")] + public static extern void PeerConnection_RegisterVideoTrackRemovedCallback(PeerConnectionHandle peerHandle, + PeerConnectionVideoTrackRemovedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRegisterRemoteAudioFrameCallback")] - public static extern void PeerConnection_RegisterRemoteAudioFrameCallback(PeerConnectionHandle peerHandle, - AudioFrameUnmanagedCallback callback, IntPtr userData); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionAddLocalVideoTrack")] - public static extern uint PeerConnection_AddLocalVideoTrack(PeerConnectionHandle peerHandle, - string trackName, in LocalVideoTrackInteropInitConfig config, out LocalVideoTrackHandle trackHandle); + EntryPoint = "mrsPeerConnectionRegisterDataChannelAddedCallback")] + public static extern void PeerConnection_RegisterDataChannelAddedCallback(PeerConnectionHandle peerHandle, + PeerConnectionDataChannelAddedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionAddLocalVideoTrackFromExternalSource")] - public static extern uint PeerConnection_AddLocalVideoTrackFromExternalSource( - PeerConnectionHandle peerHandle, string trackName, ExternalVideoTrackSourceHandle sourceHandle, - in LocalVideoTrackFromExternalSourceInteropInitConfig config, out LocalVideoTrackHandle trackHandle); + EntryPoint = "mrsPeerConnectionRegisterDataChannelRemovedCallback")] + public static extern void PeerConnection_RegisterDataChannelRemovedCallback(PeerConnectionHandle peerHandle, + PeerConnectionDataChannelRemovedCallback callback, IntPtr userData); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionAddLocalAudioTrack")] - public static extern uint PeerConnection_AddLocalAudioTrack(PeerConnectionHandle peerHandle); + EntryPoint = "mrsPeerConnectionAddTransceiver")] + public static extern uint PeerConnection_AddTransceiver(PeerConnectionHandle peerHandle, + in TransceiverInterop.InitConfig config, out IntPtr transceiverHandle); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsPeerConnectionAddDataChannel")] - public static extern uint PeerConnection_AddDataChannel(PeerConnectionHandle peerHandle, IntPtr dataChannel, - DataChannelInterop.CreateConfig config, DataChannelInterop.Callbacks callbacks, - ref IntPtr dataChannelHandle); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRemoveLocalAudioTrack")] - public static extern void PeerConnection_RemoveLocalAudioTrack(PeerConnectionHandle peerHandle); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRemoveLocalVideoTrack")] - public static extern uint PeerConnection_RemoveLocalVideoTrack(PeerConnectionHandle peerHandle, - LocalVideoTrackHandle trackHandle); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionRemoveLocalVideoTracksFromSource")] - public static extern uint PeerConnection_RemoveLocalVideoTracksFromSource(PeerConnectionHandle peerHandle, - ExternalVideoTrackSourceHandle sourceHandle); + public static extern uint PeerConnection_AddDataChannel(PeerConnectionHandle peerHandle, + ref DataChannelInterop.CreateConfig config, ref IntPtr dataChannelHandle); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsPeerConnectionRemoveDataChannel")] public static extern uint PeerConnection_RemoveDataChannel(PeerConnectionHandle peerHandle, IntPtr dataChannelHandle); - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionSetLocalAudioTrackEnabled")] - public static extern uint PeerConnection_SetLocalAudioTrackEnabled(PeerConnectionHandle peerHandle, int enabled); - - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsPeerConnectionIsLocalAudioTrackEnabled")] - public static extern int PeerConnection_IsLocalAudioTrackEnabled(PeerConnectionHandle peerHandle); - [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsPeerConnectionAddIceCandidate")] public static extern void PeerConnection_AddIceCandidate(PeerConnectionHandle peerHandle, string sdpMid, @@ -742,9 +808,7 @@ public static extern uint PeerConnection_SetRemoteDescriptionAsync(PeerConnectio #endregion - - #region Helpers - + #region Utilities class RemoteDescArgs { public ActionDelegate callback; diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteAudioTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteAudioTrackInterop.cs new file mode 100644 index 000000000..984423095 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteAudioTrackInterop.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using static Microsoft.MixedReality.WebRTC.Interop.AudioTrackInterop; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + internal class RemoteAudioTrackInterop + { + #region Native functions + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteAudioTrackSetUserData")] + public static unsafe extern void RemoteAudioTrack_SetUserData(IntPtr handle, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteAudioTrackGetUserData")] + public static unsafe extern IntPtr RemoteAudioTrack_GetUserData(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteAudioTrackRegisterFrameCallback")] + public static extern void RemoteAudioTrack_RegisterFrameCallback(IntPtr trackHandle, + AudioFrameUnmanagedCallback callback, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteAudioTrackIsEnabled")] + public static extern int RemoteAudioTrack_IsEnabled(IntPtr trackHandle); + + #endregion + + + #region Marshaling data structures + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + public struct CreateConfig + { + public string TrackName; + } + + #endregion + + + #region Native callbacks + + public class InteropCallbackArgs + { + public RemoteAudioTrack Track; + public AudioFrameUnmanagedCallback FrameCallback; + } + + [MonoPInvokeCallback(typeof(AudioFrameUnmanagedCallback))] + public static void FrameCallback(IntPtr userData, ref AudioFrame frame) + { + var track = Utils.ToWrapper(userData); + track.OnFrameReady(frame); + } + + #endregion + + + #region Utilities + + public static RemoteAudioTrack CreateWrapper(PeerConnection parent, in PeerConnectionInterop.RemoteAudioTrackAddedInfo config) + { + // Create a new wrapper + var wrapper = new RemoteAudioTrack(config.trackHandle, parent, config.trackName); + + // Assign a reference to it inside the UserData of the native object so it can be retrieved whenever needed + IntPtr wrapperRef = Utils.MakeWrapperRef(wrapper); + RemoteAudioTrack_SetUserData(config.trackHandle, wrapperRef); + + return wrapper; + } + + #endregion + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteVideoTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteVideoTrackInterop.cs new file mode 100644 index 000000000..1d9c3cbf4 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/RemoteVideoTrackInterop.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + internal class RemoteVideoTrackInterop + { + #region Native functions + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteVideoTrackSetUserData")] + public static unsafe extern void RemoteVideoTrack_SetUserData(IntPtr handle, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteVideoTrackGetUserData")] + public static unsafe extern IntPtr RemoteVideoTrack_GetUserData(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteVideoTrackRegisterI420AFrameCallback")] + public static extern void RemoteVideoTrack_RegisterI420AFrameCallback(IntPtr trackHandle, + LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback callback, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteVideoTrackRegisterArgb32FrameCallback")] + public static extern void RemoteVideoTrack_RegisterArgb32FrameCallback(IntPtr trackHandle, + LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback callback, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsRemoteVideoTrackIsEnabled")] + public static extern int RemoteVideoTrack_IsEnabled(IntPtr trackHandle); + + #endregion + + + #region Marshaling data structures + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + public struct CreateConfig + { + public string TrackName; + } + + #endregion + + + #region Native callbacks + + public class InteropCallbackArgs + { + public RemoteVideoTrack Track; + public LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback I420AFrameCallback; + public LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback Argb32FrameCallback; + } + + [MonoPInvokeCallback(typeof(LocalVideoTrackInterop.I420AVideoFrameUnmanagedCallback))] + public static void I420AFrameCallback(IntPtr userData, ref I420AVideoFrame frame) + { + var track = Utils.ToWrapper(userData); + track.OnI420AFrameReady(frame); + } + + [MonoPInvokeCallback(typeof(LocalVideoTrackInterop.Argb32VideoFrameUnmanagedCallback))] + public static void Argb32FrameCallback(IntPtr userData, ref Argb32VideoFrame frame) + { + var track = Utils.ToWrapper(userData); + track.OnArgb32FrameReady(frame); + } + + #endregion + + + #region Utilities + + public static RemoteVideoTrack CreateWrapper(PeerConnection parent, in PeerConnectionInterop.RemoteVideoTrackAddedInfo config) + { + // Create a new wrapper + var wrapper = new RemoteVideoTrack(config.trackHandle, parent, config.trackName); + + // Assign a reference to it inside the UserData of the native object so it can be retrieved whenever needed + IntPtr wrapperRef = Utils.MakeWrapperRef(wrapper); + RemoteVideoTrack_SetUserData(config.trackHandle, wrapperRef); + + return wrapper; + } + + #endregion + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/TransceiverInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/TransceiverInterop.cs new file mode 100644 index 000000000..0c0fe8fb4 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/TransceiverInterop.cs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + internal class TransceiverInterop + { + #region Native functions + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverSetUserData")] + public static unsafe extern void Transceiver_SetUserData(IntPtr handle, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverGetUserData")] + public static unsafe extern IntPtr Transceiver_GetUserData(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverRegisterStateUpdatedCallback")] + public static unsafe extern uint Transceiver_RegisterStateUpdatedCallback(IntPtr handle, + StateUpdatedDelegate callback, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverSetDirection")] + public static unsafe extern uint Transceiver_SetDirection(IntPtr handle, + Transceiver.Direction newDirection); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverSetLocalAudioTrack")] + public static unsafe extern uint Transceiver_SetLocalAudioTrack(IntPtr handle, + LocalAudioTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverSetLocalVideoTrack")] + public static unsafe extern uint Transceiver_SetLocalVideoTrack(IntPtr handle, + LocalVideoTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverGetLocalAudioTrack")] + public static unsafe extern uint Transceiver_GetLocalAudioTrack(IntPtr handle, + out LocalAudioTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverGetLocalVideoTrack")] + public static unsafe extern uint Transceiver_GetLocalVideoTrack(IntPtr handle, + out LocalVideoTrackHandle trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverGetRemoteAudioTrack")] + public static unsafe extern uint Transceiver_GetRemoteAudioTrack(IntPtr handle, out IntPtr trackHandle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsTransceiverGetRemoteVideoTrack")] + public static unsafe extern uint Transceiver_GetRemoteVideoTrack(IntPtr handle, out IntPtr trackHandle); + + #endregion + + + #region Marshaling data structures + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal ref struct InitConfig + { + /// + /// Name of the transceiver, for logging and debugging. + /// + [MarshalAs(UnmanagedType.LPStr)] + public string name; + + /// + /// Kind of media the new transceiver transports. + /// + public MediaKind mediaKind; + + /// + /// Initial desired direction. + /// + public Transceiver.Direction desiredDirection; + + /// + /// Stream IDs of the transceiver, encoded as a semi-colon separated list of IDs. + /// + [MarshalAs(UnmanagedType.LPStr)] + public string encodedStreamIds; + + public InitConfig(MediaKind mediaKind, TransceiverInitSettings settings) + { + name = settings?.Name; + this.mediaKind = mediaKind; + desiredDirection = (settings != null ? settings.InitialDesiredDirection : new TransceiverInitSettings().InitialDesiredDirection); + encodedStreamIds = Utils.EncodeTransceiverStreamIDs(settings?.StreamIDs); + } + } + + public enum StateUpdatedReason : int + { + LocalDesc, + RemoteDesc, + SetDirection + } + + public enum OptDirection : int + { + NotSet = -1, + SendReceive = 0, + SendOnly = 1, + ReceiveOnly = 2, + Inactive = 3, + } + + #endregion + + + #region Native callbacks + + public static readonly StateUpdatedDelegate StateUpdatedCallback = TransceiverStateUpdatedCallback; + + [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)] + public delegate void StateUpdatedDelegate(IntPtr transceiver, StateUpdatedReason reason, + OptDirection negotiatedDirection, Transceiver.Direction desiredDirection); + + [MonoPInvokeCallback(typeof(StateUpdatedDelegate))] + private static void TransceiverStateUpdatedCallback(IntPtr transceiver, StateUpdatedReason reason, + OptDirection negotiatedDirection, Transceiver.Direction desiredDirection) + { + var videoTranceiverWrapper = Utils.ToWrapper(transceiver); + var optDir = negotiatedDirection == OptDirection.NotSet ? null : (Transceiver.Direction?)negotiatedDirection; + videoTranceiverWrapper.OnStateUpdated(optDir, desiredDirection); + } + + #endregion + + + #region Utilities + + public static Transceiver CreateWrapper(PeerConnection parent, in PeerConnectionInterop.TransceiverAddedInfo info) + { + // Create a new wrapper + var streamIDs = info.encodedStreamIDs.Split(';'); + Transceiver wrapper = new Transceiver(info.transceiverHandle, info.mediaKind, parent, info.mlineIndex, info.name, streamIDs, info.desiredDirection); + + // Assign a reference to it inside the UserData of the native object so it can be retrieved whenever needed + IntPtr wrapperRef = Utils.MakeWrapperRef(wrapper); + Transceiver_SetUserData(info.transceiverHandle, wrapperRef); + + return wrapper; + } + + public static void RegisterCallbacks(Transceiver transceiver, out IntPtr argsRef) + { + argsRef = Utils.MakeWrapperRef(transceiver); + Transceiver_RegisterStateUpdatedCallback(transceiver._nativeHandle, StateUpdatedCallback, argsRef); + } + + public static void UnregisterCallbacks(IntPtr handle, IntPtr argsRef) + { + Utils.ReleaseWrapperRef(argsRef); + Transceiver_RegisterStateUpdatedCallback(handle, null, IntPtr.Zero); + } + + #endregion + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Library.cs b/libs/Microsoft.MixedReality.WebRTC/Library.cs index 576e5449b..57b072c70 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Library.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Library.cs @@ -36,8 +36,16 @@ public enum ShutdownOptionsFlags : uint /// /// Log with all objects still alive, to help debugging. + /// This is recommended to prevent deadlocks during shutdown. /// - LogLiveObjects = 1, + LogLiveObjects = 0x1, + + /// + /// When forcing shutdown, either because is called or + /// because the program terminates, and some objects are still alive, attempt + /// to break into the debugger. This is not available for all platforms. + /// + DebugBreakOnForceShutdown = 0x2, /// /// Default options. @@ -47,6 +55,8 @@ public enum ShutdownOptionsFlags : uint /// /// Options used when shutting down the MixedReality-WebRTC library. + /// disposed, to shutdown the internal threads and release the global resources, and allow the + /// library's module to be unloaded. /// public static ShutdownOptionsFlags ShutdownOptions { diff --git a/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs b/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs new file mode 100644 index 000000000..4ae9fe231 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.MixedReality.WebRTC.Interop; +using Microsoft.MixedReality.WebRTC.Tracing; + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Settings for adding a local audio track backed by a local audio capture device (e.g. microphone). + /// + public class LocalAudioTrackSettings + { + /// + /// Name of the track to create, as used for the SDP negotiation. + /// This name needs to comply with the requirements of an SDP token, as described in the SDP RFC + /// https://tools.ietf.org/html/rfc4566#page-43. In particular the name cannot contain spaces nor + /// double quotes ". + /// The track name can optionally be empty, in which case the implementation will create a valid + /// random track name. + /// + public string trackName = string.Empty; + } + + /// + /// Audio track sending to the remote peer audio frames originating from + /// a local track source (local microphone or other audio recording device). + /// + public class LocalAudioTrack : MediaTrack, IDisposable + { + /// + /// Enabled status of the track. If enabled, send local audio frames to the remote peer as + /// expected. If disabled, send only black frames instead. + /// + /// + /// Reading the value of this property after the track has been disposed is valid, and returns + /// false. Writing to this property after the track has been disposed throws an exception. + /// + public bool Enabled + { + get + { + return (LocalAudioTrackInterop.LocalAudioTrack_IsEnabled(_nativeHandle) != 0); + } + set + { + uint res = LocalAudioTrackInterop.LocalAudioTrack_SetEnabled(_nativeHandle, value ? -1 : 0); + Utils.ThrowOnErrorCode(res); + } + } + + /// + /// Event that occurs when a audio frame has been produced by the underlying source and is available. + /// + public event AudioFrameDelegate AudioFrameReady; + + /// + /// Handle to the native LocalAudioTrack object. + /// + /// + /// In native land this is a Microsoft::MixedReality::WebRTC::LocalAudioTrackHandle. + /// + internal LocalAudioTrackHandle _nativeHandle { get; private set; } = new LocalAudioTrackHandle(); + + /// + /// Handle to self for interop callbacks. This adds a reference to the current object, preventing + /// it from being garbage-collected. + /// + private IntPtr _selfHandle = IntPtr.Zero; + + /// + /// Callback arguments to ensure delegates registered with the native layer don't go out of scope. + /// + private LocalAudioTrackInterop.InteropCallbackArgs _interopCallbackArgs; + + /// + /// Create an audio track from a local audio capture device (microphone). + /// This does not add the track to any peer connection. Instead, the track must be added manually to + /// an audio transceiver to be attached to a peer connection and transmitted to a remote peer. + /// + /// Settings to initialize the local audio track. + /// Asynchronous task completed once the device is capturing and the track is created. + /// + /// On UWP this requires the "microphone" capability. + /// See + /// for more details. + /// + public static Task CreateFromDeviceAsync(LocalAudioTrackSettings settings = null) + { + return Task.Run(() => + { + // On UWP this cannot be called from the main UI thread, so always call it from + // a background worker thread. + + string trackName = settings?.trackName; + if (string.IsNullOrEmpty(trackName)) + { + trackName = Guid.NewGuid().ToString(); + } + + // Create interop wrappers + var track = new LocalAudioTrack(trackName); + + // Parse settings + var config = new PeerConnectionInterop.LocalAudioTrackInteropInitConfig(track, settings); + + // Create native implementation objects + uint res = LocalAudioTrackInterop.LocalAudioTrack_CreateFromDevice(in config, trackName, + out LocalAudioTrackHandle trackHandle); + Utils.ThrowOnErrorCode(res); + track.SetHandle(trackHandle); + return track; + }); + } + + // Constructor for interop-based creation; SetHandle() will be called later. + // Constructor for standalone track not associated to a peer connection. + internal LocalAudioTrack(string trackName) : base(null, trackName) + { + Transceiver = null; + } + + // Constructor for interop-based creation; SetHandle() will be called later. + // Constructor for a track associated with a peer connection. + internal LocalAudioTrack(PeerConnection peer, Transceiver transceiver, string trackName) : base(peer, trackName) + { + Debug.Assert(transceiver.MediaKind == MediaKind.Audio); + Debug.Assert(transceiver.LocalAudioTrack == null); + Transceiver = transceiver; + transceiver.LocalAudioTrack = this; + } + + internal void SetHandle(LocalAudioTrackHandle handle) + { + Debug.Assert(!handle.IsClosed); + // Either first-time assign or no-op (assign same value again) + Debug.Assert(_nativeHandle.IsInvalid || (_nativeHandle == handle)); + if (_nativeHandle != handle) + { + _nativeHandle = handle; + RegisterInteropCallbacks(); + } + } + + private void RegisterInteropCallbacks() + { + _interopCallbackArgs = new LocalAudioTrackInterop.InteropCallbackArgs() + { + Track = this, + FrameCallback = LocalAudioTrackInterop.FrameCallback, + }; + _selfHandle = Utils.MakeWrapperRef(this); + LocalAudioTrackInterop.LocalAudioTrack_RegisterFrameCallback( + _nativeHandle, _interopCallbackArgs.FrameCallback, _selfHandle); + } + + /// + public void Dispose() + { + if (_nativeHandle.IsClosed) + { + return; + } + + // Remove the track from the peer connection, if any + if (Transceiver != null) + { + Debug.Assert(PeerConnection != null); + Debug.Assert(Transceiver.LocalTrack == this); + Transceiver.LocalAudioTrack = null; + } + Debug.Assert(PeerConnection == null); + Debug.Assert(Transceiver == null); + + // Unregister interop callbacks + if (_selfHandle != IntPtr.Zero) + { + LocalAudioTrackInterop.LocalAudioTrack_RegisterFrameCallback(_nativeHandle, null, IntPtr.Zero); + Utils.ReleaseWrapperRef(_selfHandle); + _selfHandle = IntPtr.Zero; + _interopCallbackArgs = null; + } + + // Destroy the native object. This may be delayed if a P/Invoke callback is underway, + // but will be handled at some point anyway, even if the managed instance is gone. + _nativeHandle.Dispose(); + } + + internal void OnFrameReady(AudioFrame frame) + { + MainEventSource.Log.LocalAudioFrameReady(frame.bitsPerSample, frame.sampleRate, frame.channelCount, frame.sampleCount); + AudioFrameReady?.Invoke(frame); + } + + internal override void OnMute(bool muted) + { + + } + + /// + public override string ToString() + { + return $"(LocalAudioTrack)\"{Name}\""; + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/LocalVideoTrack.cs b/libs/Microsoft.MixedReality.WebRTC/LocalVideoTrack.cs index e4ddfd40d..77a583310 100644 --- a/libs/Microsoft.MixedReality.WebRTC/LocalVideoTrack.cs +++ b/libs/Microsoft.MixedReality.WebRTC/LocalVideoTrack.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Threading.Tasks; using Microsoft.MixedReality.WebRTC.Interop; using Microsoft.MixedReality.WebRTC.Tracing; @@ -158,24 +159,14 @@ public class LocalVideoTrackSettings /// Video track sending to the remote peer video frames originating from /// a local track source. /// - public class LocalVideoTrack : IDisposable + public class LocalVideoTrack : MediaTrack, IDisposable { /// - /// Peer connection this video track is added to, if any. - /// This is null after the track has been removed from the peer connection. + /// External source for this video track, or null if the source is some + /// internal video capture device, or has been removed from any peer connection + /// (and is therefore inactive). /// - public PeerConnection PeerConnection { get; private set; } - - /// - /// Track name as specified during creation. This property is immutable. - /// - public string Name { get; } - - /// - /// External source for this video track, or null if the source is - /// some internal video capture device. - /// - public ExternalVideoTrackSource Source { get; } = null; + public ExternalVideoTrackSource Source { get; private set; } = null; /// /// Enabled status of the track. If enabled, send local video frames to the remote peer as @@ -227,13 +218,147 @@ public bool Enabled /// private LocalVideoTrackInterop.InteropCallbackArgs _interopCallbackArgs; + /// + /// Create a video track from a local video capture device (webcam). + /// + /// The video track receives its video data from an underlying hidden source associated with + /// the track and producing video frames by capturing them from a capture device accessible + /// from the local host machine, generally a USB webcam or built-in device camera. + /// + /// The underlying video source initially starts in the capturing state, and will remain live + /// for as long as the track is alive. It can be added to a peer connection by assigning it to + /// the property of a video transceiver of that peer + /// connection. Once attached to the peer connection, it can temporarily be disabled and re-enabled + /// (see ) while remaining attached to it. + /// + /// Note that disabling the track does not release the device; the source retains exclusive access to it. + /// Therefore in general multiple tracks cannot be created using a single video capture device. + /// + /// Video capture settings for configuring the capture device associated with + /// the underlying video track source. + /// This returns a task which, upon successful completion, provides an instance of + /// representing the newly created video track. + /// + /// On UWP this requires the "webcam" capability. + /// See + /// for more details. + /// + /// The video capture device may be accessed several times during the initializing process, + /// generally once for listing and validating the capture format, and once for actually starting + /// the video capture. + /// + /// Note that the capture device must support a capture format with the given constraints of profile + /// ID or kind, capture resolution, and framerate, otherwise the call will fail. That is, there is no + /// fallback mechanism selecting a closest match. Developers should use + /// to list the supported formats ahead + /// of calling , and can build their own + /// fallback mechanism on top of this call if needed. + /// + /// The peer connection is not intialized. + /// + /// Create a video track called "MyTrack", with Mixed Reality Capture (MRC) enabled. + /// This assumes that the platform supports MRC. Note that if MRC is not available + /// the call will still succeed, but will return a track without MRC enabled. + /// + /// var settings = new LocalVideoTrackSettings + /// { + /// trackName = "MyTrack", + /// enableMrc = true + /// }; + /// var videoTrack = await LocalVideoTrack.CreateFromDeviceAsync(settings); + /// + /// Create a video track from a local webcam, asking for a capture format suited for video conferencing, + /// and a target framerate of 30 frames per second (FPS). The implementation will select an appropriate + /// capture resolution. This assumes that the device supports video profiles, and has at least one capture + /// format supporting 30 FPS capture associated with the VideoConferencing profile. Otherwise the call + /// will fail. + /// + /// var settings = new LocalVideoTrackSettings + /// { + /// videoProfileKind = VideoProfileKind.VideoConferencing, + /// framerate = 30.0 + /// }; + /// var videoTrack = await LocalVideoTrack.CreateFromDeviceAsync(settings); + /// + /// + /// + public static Task CreateFromDeviceAsync(LocalVideoTrackSettings settings = null) + { + return Task.Run(() => + { + // On UWP this cannot be called from the main UI thread, so always call it from + // a background worker thread. + + string trackName = settings?.trackName; + if (string.IsNullOrEmpty(trackName)) + { + trackName = Guid.NewGuid().ToString(); + } + + // Create interop wrappers + var track = new LocalVideoTrack(trackName); + + // Parse settings + var config = new PeerConnectionInterop.LocalVideoTrackInteropInitConfig(track, settings); + + // Create native implementation objects + uint res = LocalVideoTrackInterop.LocalVideoTrack_CreateFromDevice(in config, trackName, + out LocalVideoTrackHandle trackHandle); + Utils.ThrowOnErrorCode(res); + track.SetHandle(trackHandle); + return track; + }); + } + + /// + /// Create a new local video track backed by an existing external video source. + /// The track can be added to a peer connection by setting the + /// property. + /// + /// Name of the new local video track. + /// External video track source providing some video frames to the track. + /// The newly created local video track. + /// + public static LocalVideoTrack CreateFromExternalSource(string trackName, ExternalVideoTrackSource source) + { + if (string.IsNullOrEmpty(trackName)) + { + trackName = Guid.NewGuid().ToString(); + } + + // Create interop wrappers + var track = new LocalVideoTrack(trackName, source); + + // Parse settings + var config = new PeerConnectionInterop.LocalVideoTrackFromExternalSourceInteropInitConfig(trackName, source); + + // Create native implementation objects + uint res = LocalVideoTrackInterop.LocalVideoTrack_CreateFromExternalSource(in config, out LocalVideoTrackHandle trackHandle); + Utils.ThrowOnErrorCode(res); + track.SetHandle(trackHandle); + return track; + } + + // Constructor for interop-based creation; SetHandle() will be called later. + // Constructor for standalone track not associated to a peer connection. + internal LocalVideoTrack(string trackName, ExternalVideoTrackSource source = null) : base(null, trackName) + { + Transceiver = null; + Source = source; + source?.OnTrackAddedToSource(this); + } + // Constructor for interop-based creation; SetHandle() will be called later // Constructor for a track associated with a peer connection. - internal LocalVideoTrack(PeerConnection peer, string trackName, ExternalVideoTrackSource source = null) + internal LocalVideoTrack(PeerConnection peer, + Transceiver transceiver, string trackName, ExternalVideoTrackSource source = null) : base(peer, trackName) { - PeerConnection = peer; - Name = trackName; + Debug.Assert(transceiver.MediaKind == MediaKind.Video); + Debug.Assert(transceiver.LocalVideoTrack == null); + Transceiver = transceiver; + transceiver.LocalVideoTrack = this; Source = source; + source?.OnTrackAddedToSource(this); } internal void SetHandle(LocalVideoTrackHandle handle) @@ -271,9 +396,22 @@ public void Dispose() return; } + // Notify the source + if (Source != null) + { + Source.OnTrackRemovedFromSource(this); + Source = null; + } + // Remove the track from the peer connection, if any - PeerConnection?.RemoveLocalVideoTrack(this); - Debug.Assert(PeerConnection == null); // see OnTrackRemoved + if (Transceiver != null) + { + Debug.Assert(PeerConnection != null); + Debug.Assert(Transceiver.LocalTrack == this); + Transceiver.LocalVideoTrack = null; + } + Debug.Assert(PeerConnection == null); + Debug.Assert(Transceiver == null); // Unregister interop callbacks if (_selfHandle != IntPtr.Zero) @@ -285,10 +423,6 @@ public void Dispose() _interopCallbackArgs = null; } - // Currently there is a 1:1 mapping between track and source, so the track owns its - // source and must dipose of it. - Source?.Dispose(); - // Destroy the native object. This may be delayed if a P/Invoke callback is underway, // but will be handled at some point anyway, even if the managed instance is gone. _nativeHandle.Dispose(); @@ -306,11 +440,15 @@ internal void OnArgb32FrameReady(Argb32VideoFrame frame) Argb32VideoFrameReady?.Invoke(frame); } - internal void OnTrackRemoved(PeerConnection previousConnection) + internal override void OnMute(bool muted) + { + + } + + /// + public override string ToString() { - Debug.Assert(PeerConnection == previousConnection); - Debug.Assert(!_nativeHandle.IsClosed); - PeerConnection = null; + return $"(LocalVideoTrack)\"{Name}\""; } } } diff --git a/libs/Microsoft.MixedReality.WebRTC/MediaTrack.cs b/libs/Microsoft.MixedReality.WebRTC/MediaTrack.cs new file mode 100644 index 000000000..153cb676c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/MediaTrack.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Base class for media tracks sending to or receiving from the remote peer. + /// + public abstract class MediaTrack + { + /// + /// Transceiver this track is attached to, if any. + /// + public Transceiver Transceiver { get; protected internal set; } + + /// + /// Peer connection this media track is added to, if any. + /// This is null after the track has been removed from the peer connection. + /// + public PeerConnection PeerConnection { get; protected internal set; } + + /// + /// Track name as specified during creation. This property is immutable. + /// For remote tracks the property is specified by the remote peer. + /// + public string Name { get; } + + internal MediaTrack(PeerConnection peer, string trackName) + { + PeerConnection = peer; + Name = trackName; + } + + internal abstract void OnMute(bool muted); + + /// + public override string ToString() + { + return $"(MediaTrack)\"{Name}\""; + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs b/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs index 9e31b6349..c9c60e18f 100644 --- a/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs +++ b/libs/Microsoft.MixedReality.WebRTC/PeerConnection.cs @@ -280,11 +280,193 @@ public struct VideoCaptureFormat public uint fourcc; } + /// + /// Settings to create a new transceiver wrapper. + /// + public class TransceiverInitSettings + { + /// + /// Transceiver name, for logging and debugging. + /// + public string Name; + + /// + /// Initial value of . + /// + public Transceiver.Direction InitialDesiredDirection = Transceiver.Direction.SendReceive; + + /// + /// List of stream IDs to associate the transceiver with. + /// + public List StreamIDs; + } + + /// + /// Wrapper for an event possibly delayed. + /// + internal class DelayedEvent + { + /// + /// The event handler. + /// + public Action Event; + + /// + /// Begin suspending . This must be matched with a call + /// to . During this time, calling + /// does not invoke the event but instead queue it for later invoking by + /// . + /// + public void BeginSuspend() + { + lock (_lock) + { + ++_suspendCount; + } + } + + /// + /// End suspending and invoke it if any call to + /// was made since the first call. + /// + public void EndSuspend() + { + lock (_lock) + { + Debug.Assert(_suspendCount > 0); + --_suspendCount; + if (!_eventPending || (_suspendCount > 0)) + { + return; + } + _eventPending = false; + } + Event?.Invoke(); + } + + /// + /// Try to invoke , either immediately if not suspended, or later + /// when the last call stops suspending it. + /// + /// If the event is not suspended, and therefore is invoked immediately, + /// then invoke it asynchronously from a worker thread. + public void Invoke(bool async = false) + { + lock (_lock) + { + Debug.Assert(_suspendCount >= 0); + if (_suspendCount > 0) + { + _eventPending = true; + return; + } + } + if (async) + { + Task.Run(Event); + } + else + { + Event.Invoke(); + } + } + + /// + /// Try to invoke asynchronously. + /// This is equivalent to Invoke(async: true). + /// + public void InvokeAsync() => Invoke(async: true); + + /// + /// Lock for internal variables: + /// - + /// - + /// + private object _lock = new object(); + + /// + /// Number of concurrent calls currently suspending the event. + /// When this value reaches zero, the thread which decremented it checks the value of + /// and if true then invoke the event. + /// + /// This is protected by . + private int _suspendCount = 0; + + /// + /// Was any event internally raised while the public event was suspended (that is, + /// was non-zero)? This is set by and cleared by any thread actually invoking the event after + /// decrementing to zero. + /// + /// This is protected by . + private bool _eventPending = false; + } + + /// + /// RAII helper to start/stop a delay block for a . + /// + internal class ScopedDelayedEvent : IDisposable + { + /// + /// Initialize a scoped delay for the specified . + /// This will call immediately, and will call + /// once disposed. + /// + /// + public ScopedDelayedEvent(DelayedEvent ev) + { + _delayedEvent = ev; + _delayedEvent.BeginSuspend(); + } + + /// + /// Dispose of the helper and call on + /// . + /// + public void Dispose() + { + _delayedEvent.EndSuspend(); + } + + private DelayedEvent _delayedEvent; + } + /// /// The WebRTC peer connection object is the entry point to using WebRTC. /// public class PeerConnection : IDisposable { + /// + /// Delegate for event. + /// + /// The newly added transceiver. + public delegate void TransceiverAddedDelegate(Transceiver transceiver); + + /// + /// Delegate for event. + /// + /// The newly added audio track. + public delegate void AudioTrackAddedDelegate(RemoteAudioTrack track); + + /// + /// Delegate for event. + /// + /// The audio transceiver the track was removed from. + /// The audio track just removed. + public delegate void AudioTrackRemovedDelegate(Transceiver transceiver, RemoteAudioTrack track); + + /// + /// Delegate for event. + /// + /// The newly added video track. + public delegate void VideoTrackAddedDelegate(RemoteVideoTrack track); + + /// + /// Delegate for event. + /// + /// The video transceiver the track was removed from. + /// The video track just removed. + public delegate void VideoTrackRemovedDelegate(Transceiver transceiver, RemoteVideoTrack track); + /// /// Delegate for event. /// @@ -424,6 +606,11 @@ public enum TrackKind : uint #endregion + /// + /// A name for the peer connection, used for logging and debugging. + /// + public string Name { get; set; } = string.Empty; + /// /// Boolean property indicating whether the peer connection has been initialized. /// @@ -449,6 +636,96 @@ public bool Initialized /// public bool IsConnected { get; private set; } = false; + /// + /// Collection of transceivers for the peer conneciton. + /// Transceivers are present in the list in order of their media line index + /// (). Once a transceiver is added to the + /// peer connection, it cannot be removed, but its tracks can be changed. + /// This requires some renegotiation. + /// + public List Transceivers { get; } = new List(); + + /// + /// Collection of local audio tracks attached to the peer connection. + /// + public IEnumerable LocalAudioTracks + { + get + { + foreach (var tr in Transceivers) + { + var audioTrack = tr.LocalAudioTrack; + if (audioTrack != null) + { + yield return audioTrack; + } + } + } + } + + /// + /// Collection of local video tracks attached to the peer connection. + /// + public IEnumerable LocalVideoTracks + { + get + { + foreach (var tr in Transceivers) + { + var videoTrack = tr.LocalVideoTrack; + if (videoTrack != null) + { + yield return videoTrack; + } + } + } + } + + /// + /// Collection of remote audio tracks attached to the peer connection. + /// + public IEnumerable RemoteAudioTracks + { + get + { + foreach (var tr in Transceivers) + { + var audioTrack = tr.RemoteAudioTrack; + if (audioTrack != null) + { + yield return audioTrack; + } + } + } + } + + /// + /// Collection of remote video tracks attached to the peer connection. + /// + public IEnumerable RemoteVideoTracks + { + get + { + foreach (var tr in Transceivers) + { + var videoTrack = tr.RemoteVideoTrack; + if (videoTrack != null) + { + yield return videoTrack; + } + } + } + } + + /// + /// Collection of data channels for the peer connection. + /// + /// Data channels are either manually added with + /// or , or are created by the implementation + /// when the remote peer creates a new in-band data channel. + /// + public List DataChannels { get; } = new List(); + /// /// Event fired when a connection is established. /// @@ -497,45 +774,42 @@ public bool Initialized /// and the user should call to actually /// start a renegotiation. /// - public event Action RenegotiationNeeded; - - /// - /// Event that occurs when a remote track is added to the current connection. - /// - public event Action TrackAdded; + public event Action RenegotiationNeeded + { + add => _renegotiationNeededEvent.Event += value; + remove => _renegotiationNeededEvent.Event -= value; + } /// - /// Event that occurs when a remote track is removed from the current connection. + /// Event that occurs when a transceiver is added to the peer connection, either + /// manually using , or + /// automatically as a result of a new session negotiation. /// - public event Action TrackRemoved; + /// + /// Transceivers cannot be removed from the peer connection, so there is no + /// TransceiverRemoved event. + /// + public event TransceiverAddedDelegate TransceiverAdded; /// - /// Event that occurs when a video frame from a remote peer has been - /// received and is available for render. + /// Event that occurs when a remote audio track is added to the current connection. /// - public event I420AVideoFrameDelegate I420ARemoteVideoFrameReady; + public event AudioTrackAddedDelegate AudioTrackAdded; /// - /// Event that occurs when a video frame from a remote peer has been - /// received and is available for render. + /// Event that occurs when a remote audio track is removed from the current connection. /// - public event Argb32VideoFrameDelegate Argb32RemoteVideoFrameReady; + public event AudioTrackRemovedDelegate AudioTrackRemoved; /// - /// Event that occurs when an audio frame from a local track has been - /// produced locally and is available for render. + /// Event that occurs when a remote video track is added to the current connection. /// - /// - /// WARNING -- This is currently not implemented in the underlying WebRTC - /// implementation, so THIS EVENT IS NEVER FIRED. - /// - public event AudioFrameDelegate LocalAudioFrameReady; + public event VideoTrackAddedDelegate VideoTrackAdded; /// - /// Event that occurs when an audio frame from a remote peer has been - /// received and is available for render. + /// Event that occurs when a remote video track is removed from the current connection. /// - public event AudioFrameDelegate RemoteAudioFrameReady; + public event VideoTrackRemovedDelegate VideoTrackRemoved; /// /// GCHandle to self for the various native callbacks. @@ -569,9 +843,24 @@ public bool Initialized /// private object _openCloseLock = new object(); - private PeerConnectionInterop.InteropCallbacks _interopCallbacks; private PeerConnectionInterop.PeerCallbackArgs _peerCallbackArgs; + /// + /// Lock for accessing the collections of tracks and transceivers: + /// - + /// - + /// - + /// - + /// - + /// + private object _tracksLock = new object(); + + /// + /// Implementation of adding the capability to delay the event. + /// This allows to wait until the + /// newly created transceiver wrapper is fully instantiated to dispatch the event. + /// + private DelayedEvent _renegotiationNeededEvent = new DelayedEvent(); #region Initializing and shutdown @@ -626,11 +915,6 @@ public Task InitializeAsync(PeerConnectionConfiguration config = null, Cancellat // Create and lock in memory delegates for all the static callback wrappers (see below). // This avoids delegates being garbage-collected, since the P/Invoke mechanism by itself // does not guarantee their lifetime. - _interopCallbacks = new PeerConnectionInterop.InteropCallbacks() - { - Peer = this, - DataChannelCreateObjectCallback = DataChannelInterop.DataChannelCreateObjectCallback, - }; _peerCallbackArgs = new PeerConnectionInterop.PeerCallbackArgs() { Peer = this, @@ -642,40 +926,39 @@ public Task InitializeAsync(PeerConnectionConfiguration config = null, Cancellat IceStateChangedCallback = PeerConnectionInterop.IceStateChangedCallback, IceGatheringStateChangedCallback = PeerConnectionInterop.IceGatheringStateChangedCallback, RenegotiationNeededCallback = PeerConnectionInterop.RenegotiationNeededCallback, - TrackAddedCallback = PeerConnectionInterop.TrackAddedCallback, - TrackRemovedCallback = PeerConnectionInterop.TrackRemovedCallback, - I420ARemoteVideoFrameCallback = PeerConnectionInterop.I420ARemoteVideoFrameCallback, - Argb32RemoteVideoFrameCallback = PeerConnectionInterop.Argb32RemoteVideoFrameCallback, - LocalAudioFrameCallback = PeerConnectionInterop.LocalAudioFrameCallback, - RemoteAudioFrameCallback = PeerConnectionInterop.RemoteAudioFrameCallback + TransceiverAddedCallback = PeerConnectionInterop.TransceiverAddedCallback, + AudioTrackAddedCallback = PeerConnectionInterop.AudioTrackAddedCallback, + AudioTrackRemovedCallback = PeerConnectionInterop.AudioTrackRemovedCallback, + VideoTrackAddedCallback = PeerConnectionInterop.VideoTrackAddedCallback, + VideoTrackRemovedCallback = PeerConnectionInterop.VideoTrackRemovedCallback, }; - // Cache values in local variables before starting async task, to avoid any - // subsequent external change from affecting that task. - // Also set default values, as the native call doesn't handle NULL. - PeerConnectionInterop.PeerConnectionConfiguration nativeConfig; - if (config != null) - { - nativeConfig = new PeerConnectionInterop.PeerConnectionConfiguration - { - EncodedIceServers = string.Join("\n\n", config.IceServers), - IceTransportType = config.IceTransportType, - BundlePolicy = config.BundlePolicy, - SdpSemantic = config.SdpSemantic, - }; - } - else - { - nativeConfig = new PeerConnectionInterop.PeerConnectionConfiguration(); - } - // On UWP PeerConnectionCreate() fails on main UI thread, so always initialize the native peer // connection asynchronously from a background worker thread. _initTask = Task.Run(() => { token.ThrowIfCancellationRequested(); - uint res = PeerConnectionInterop.PeerConnection_Create(nativeConfig, GCHandle.ToIntPtr(_selfHandle), out _nativePeerhandle); + // Cache values in local variables before starting async task, to avoid any + // subsequent external change from affecting that task. + // Also set default values, as the native call doesn't handle NULL. + PeerConnectionInterop.PeerConnectionConfiguration nativeConfig; + if (config != null) + { + nativeConfig = new PeerConnectionInterop.PeerConnectionConfiguration + { + EncodedIceServers = string.Join("\n\n", config.IceServers), + IceTransportType = config.IceTransportType, + BundlePolicy = config.BundlePolicy, + SdpSemantic = config.SdpSemantic, + }; + } + else + { + nativeConfig = new PeerConnectionInterop.PeerConnectionConfiguration(); + } + + uint res = PeerConnectionInterop.PeerConnection_Create(ref nativeConfig, out _nativePeerhandle); lock (_openCloseLock) { @@ -684,7 +967,6 @@ public Task InitializeAsync(PeerConnectionConfiguration config = null, Cancellat { if (_selfHandle.IsAllocated) { - _interopCallbacks = null; _peerCallbackArgs = null; _selfHandle.Free(); } @@ -717,12 +999,6 @@ public Task InitializeAsync(PeerConnectionConfiguration config = null, Cancellat // Since the current PeerConnection instance is already locked via _selfHandle, // and it references all delegates via _peerCallbackArgs, those also can't be GC'd. var self = GCHandle.ToIntPtr(_selfHandle); - var interopCallbacks = new PeerConnectionInterop.MarshaledInteropCallbacks - { - DataChannelCreateObjectCallback = _interopCallbacks.DataChannelCreateObjectCallback - }; - PeerConnectionInterop.PeerConnection_RegisterInteropCallbacks( - _nativePeerhandle, in interopCallbacks); PeerConnectionInterop.PeerConnection_RegisterConnectedCallback( _nativePeerhandle, _peerCallbackArgs.ConnectedCallback, self); PeerConnectionInterop.PeerConnection_RegisterLocalSdpReadytoSendCallback( @@ -735,22 +1011,20 @@ public Task InitializeAsync(PeerConnectionConfiguration config = null, Cancellat _nativePeerhandle, _peerCallbackArgs.IceGatheringStateChangedCallback, self); PeerConnectionInterop.PeerConnection_RegisterRenegotiationNeededCallback( _nativePeerhandle, _peerCallbackArgs.RenegotiationNeededCallback, self); - PeerConnectionInterop.PeerConnection_RegisterTrackAddedCallback( - _nativePeerhandle, _peerCallbackArgs.TrackAddedCallback, self); - PeerConnectionInterop.PeerConnection_RegisterTrackRemovedCallback( - _nativePeerhandle, _peerCallbackArgs.TrackRemovedCallback, self); + PeerConnectionInterop.PeerConnection_RegisterTransceiverAddedCallback( + _nativePeerhandle, _peerCallbackArgs.TransceiverAddedCallback, self); + PeerConnectionInterop.PeerConnection_RegisterAudioTrackAddedCallback( + _nativePeerhandle, _peerCallbackArgs.AudioTrackAddedCallback, self); + PeerConnectionInterop.PeerConnection_RegisterAudioTrackRemovedCallback( + _nativePeerhandle, _peerCallbackArgs.AudioTrackRemovedCallback, self); + PeerConnectionInterop.PeerConnection_RegisterVideoTrackAddedCallback( + _nativePeerhandle, _peerCallbackArgs.VideoTrackAddedCallback, self); + PeerConnectionInterop.PeerConnection_RegisterVideoTrackRemovedCallback( + _nativePeerhandle, _peerCallbackArgs.VideoTrackRemovedCallback, self); PeerConnectionInterop.PeerConnection_RegisterDataChannelAddedCallback( _nativePeerhandle, _peerCallbackArgs.DataChannelAddedCallback, self); PeerConnectionInterop.PeerConnection_RegisterDataChannelRemovedCallback( _nativePeerhandle, _peerCallbackArgs.DataChannelRemovedCallback, self); - PeerConnectionInterop.PeerConnection_RegisterI420ARemoteVideoFrameCallback( - _nativePeerhandle, _peerCallbackArgs.I420ARemoteVideoFrameCallback, self); - PeerConnectionInterop.PeerConnection_RegisterArgb32RemoteVideoFrameCallback( - _nativePeerhandle, _peerCallbackArgs.Argb32RemoteVideoFrameCallback, self); - PeerConnectionInterop.PeerConnection_RegisterLocalAudioFrameCallback( - _nativePeerhandle, _peerCallbackArgs.LocalAudioFrameCallback, self); - PeerConnectionInterop.PeerConnection_RegisterRemoteAudioFrameCallback( - _nativePeerhandle, _peerCallbackArgs.RemoteAudioFrameCallback, self); } }, token); @@ -784,10 +1058,7 @@ public void Close() _initTask = null; // This marks the Initialized property as false IsConnected = false; - // Unregister all callbacks and free the delegates - var interopCallbacks = new PeerConnectionInterop.MarshaledInteropCallbacks(); - PeerConnectionInterop.PeerConnection_RegisterInteropCallbacks( - _nativePeerhandle, in interopCallbacks); + // Unregister all callbacks that could add some new objects PeerConnectionInterop.PeerConnection_RegisterConnectedCallback( _nativePeerhandle, null, IntPtr.Zero); PeerConnectionInterop.PeerConnection_RegisterLocalSdpReadytoSendCallback( @@ -800,28 +1071,14 @@ public void Close() _nativePeerhandle, null, IntPtr.Zero); PeerConnectionInterop.PeerConnection_RegisterRenegotiationNeededCallback( _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterTrackAddedCallback( - _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterTrackRemovedCallback( - _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterDataChannelAddedCallback( - _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterDataChannelRemovedCallback( + PeerConnectionInterop.PeerConnection_RegisterTransceiverAddedCallback( _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterI420ARemoteVideoFrameCallback( + PeerConnectionInterop.PeerConnection_RegisterAudioTrackAddedCallback( _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterArgb32RemoteVideoFrameCallback( + PeerConnectionInterop.PeerConnection_RegisterVideoTrackAddedCallback( _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterLocalAudioFrameCallback( - _nativePeerhandle, null, IntPtr.Zero); - PeerConnectionInterop.PeerConnection_RegisterRemoteAudioFrameCallback( + PeerConnectionInterop.PeerConnection_RegisterDataChannelAddedCallback( _nativePeerhandle, null, IntPtr.Zero); - if (_selfHandle.IsAllocated) - { - _interopCallbacks = null; - _peerCallbackArgs = null; - _selfHandle.Free(); - } } // Wait for any pending initializing to finish. @@ -830,15 +1087,35 @@ public void Close() initTask.Wait(); // Close the native peer connection, disconnecting from the remote peer if currently connected. + // This will invalidate all handles to transceivers and remote tracks, as the corresponding + // native objects will be destroyed. PeerConnectionInterop.PeerConnection_Close(_nativePeerhandle); // Destroy the native peer connection object. This may be delayed if a P/Invoke callback is underway, // but will be handled at some point anyway, even if the PeerConnection managed instance is gone. _nativePeerhandle.Close(); + // Notify owned objects to perform clean-up. + lock (_tracksLock) + { + foreach (var transceiver in Transceivers) + { + transceiver?.CleanUpAfterNativeDestroyed(); + } + Transceivers.Clear(); + } + // Complete shutdown sequence and re-enable InitializeAsync() lock (_openCloseLock) { + // Free all delegates for callbacks previously registered with the native + // peer connection, which is now destroyed. + if (_selfHandle.IsAllocated) + { + _peerCallbackArgs = null; + _selfHandle.Free(); + } + _isClosing = false; } } @@ -853,228 +1130,62 @@ public void Close() #endregion - #region Local audio and video tracks + #region Transceivers /// - /// Add to the current connection a video track from a local video capture device (webcam). - /// - /// The video track receives its video data from an underlying hidden source associated with - /// the track and producing video frames by capturing them from a capture device accessible - /// from the local host machine, generally a USB webcam or built-in device camera. - /// - /// The underlying video source initially starts in the capturing state, and will remain live - /// for as long as the track is added to the peer connection. It can be temporarily disabled - /// and re-enabled (see ) while remaining added to the - /// peer connection. Note that disabling the track does not release the device; the source - /// retains exclusive access to it. - /// - /// Video capture settings for configuring the capture device associated with - /// the underlying video track source. - /// This returns a task which, upon successful completion, provides an instance of - /// representing the newly added video track. - /// - /// On UWP this requires the "webcam" capability. - /// See - /// for more details. - /// - /// The video capture device may be accessed several times during the initializing process, - /// generally once for listing and validating the capture format, and once for actually starting - /// the video capture. - /// - /// Note that the capture device must support a capture format with the given constraints of profile - /// ID or kind, capture resolution, and framerate, otherwise the call will fail. That is, there is no - /// fallback mechanism selecting a closest match. Developers should use - /// to list the supported formats ahead of calling - /// , and can build their own fallback - /// mechanism on top of this call if needed. - /// - /// The peer connection is not initialized. - /// - /// Create a video track called "MyTrack", with Mixed Reality Capture (MRC) enabled. - /// This assumes that the platform supports MRC. Note that if MRC is not available - /// the call will still succeed, but will return a track without MRC enabled. - /// - /// var settings = new LocalVideoTrackSettings - /// { - /// trackName = "MyTrack", - /// enableMrc = true - /// }; - /// var videoTrack = await peerConnection.AddLocalVideoTrackAsync(settings); - /// - /// Create a video track from a local webcam, asking for a capture format suited for video conferencing, - /// and a target framerate of 30 frames per second (FPS). The implementation will select an appropriate - /// capture resolution. This assumes that the device supports video profiles, and has at least one capture - /// format supporting 30 FPS capture associated with the VideoConferencing profile. Otherwise the call - /// will fail. - /// - /// var settings = new LocalVideoTrackSettings - /// { - /// videoProfileKind = VideoProfileKind.VideoConferencing, - /// framerate = 30.0 - /// }; - /// var videoTrack = await peerConnection.AddLocalVideoTrackAsync(settings); - /// - /// - public Task AddLocalVideoTrackAsync(LocalVideoTrackSettings settings = default) - { - ThrowIfConnectionNotOpen(); - return Task.Run(() => - { - // On UWP this cannot be called from the main UI thread, so always call it from - // a background worker thread. - - string trackName = settings?.trackName; - if (string.IsNullOrEmpty(trackName)) - { - trackName = Guid.NewGuid().ToString(); - } - - // Create interop wrappers - var trackWrapper = new LocalVideoTrack(this, trackName); - - // Parse settings - var config = new PeerConnectionInterop.LocalVideoTrackInteropInitConfig(trackWrapper, settings); - - // Create native implementation objects - uint res = PeerConnectionInterop.PeerConnection_AddLocalVideoTrack(_nativePeerhandle, trackName, in config, - out LocalVideoTrackHandle trackHandle); - Utils.ThrowOnErrorCode(res); - trackWrapper.SetHandle(trackHandle); - return trackWrapper; - }); - } - - /// - /// Remove from the current connection the local video track added with . - /// - /// The peer connection is not initialized. - public void RemoveLocalVideoTrack(LocalVideoTrack track) - { - ThrowIfConnectionNotOpen(); - PeerConnectionInterop.PeerConnection_RemoveLocalVideoTrack(_nativePeerhandle, track._nativeHandle); - track.OnTrackRemoved(this); // LocalVideoTrack.PeerConnection = null - } - - /// - /// Add a local video track backed by an external video source managed by the caller. - /// Unlike with which manages - /// a local video capture device and automatically produce frames, an external video source - /// provides video frames directly to WebRTC when asked to do so via the provided callback. + /// Add to the current connection a new media transceiver. + /// + /// A transceiver is a container for a pair of media tracks, one local sending to the remote + /// peer, and one remote receiving from the remote peer. Both are optional, and the transceiver + /// can be in receive-only mode (no local track), in send-only mode (no remote track), or + /// inactive (neither local nor remote track). + /// + /// Once a transceiver is added to the peer connection, it cannot be removed, but its tracks can be + /// changed (this requires some renegotiation). /// - /// Name of the new track. - /// External source providing the frames for the track. - public LocalVideoTrack AddCustomLocalVideoTrack(string trackName, ExternalVideoTrackSource source) + /// Kind of media the transeiver is transporting. + /// Settings to initialize the new transceiver. + /// The newly created transceiver. + public Transceiver AddTransceiver(MediaKind mediaKind, TransceiverInitSettings settings = null) { ThrowIfConnectionNotOpen(); - if (string.IsNullOrEmpty(trackName)) + // Suspend RenegotiationNeeded event while creating the transceiver, to avoid firing an event + // while the transceiver is in an intermediate state. + using (var renegotiationNeeded = new ScopedDelayedEvent(_renegotiationNeededEvent)) { - trackName = Guid.NewGuid().ToString(); - } - - // Create interop wrappers - var trackWrapper = new LocalVideoTrack(this, trackName, source); - - // Parse settings - var config = new PeerConnectionInterop.LocalVideoTrackFromExternalSourceInteropInitConfig(trackWrapper, source); - - // Create native implementation objects - uint res = PeerConnectionInterop.PeerConnection_AddLocalVideoTrackFromExternalSource(_nativePeerhandle, trackName, - source._nativeHandle, in config, out LocalVideoTrackHandle trackHandle); - Utils.ThrowOnErrorCode(res); - trackWrapper.SetHandle(trackHandle); - source.OnTracksAddedToSource(this); - return trackWrapper; - } - - /// - /// Add to the current connection an audio track from a local audio capture device (microphone). - /// - /// Asynchronous task completed once the device is capturing and the track is added. - /// - /// On UWP this requires the "microphone" capability. - /// See - /// for more details. - /// - /// The peer connection is not initialized. - public Task AddLocalAudioTrackAsync() - { - ThrowIfConnectionNotOpen(); - return Task.Run(() => - { - // On UWP this cannot be called from the main UI thread, so always call it from - // a background worker thread. - if (PeerConnectionInterop.PeerConnection_AddLocalAudioTrack(_nativePeerhandle) != Utils.MRS_SUCCESS) - { - throw new Exception(); - } - }); - } - - /// - /// Remove all the local video tracks associated with the given video track source. - /// - /// The video track source. - /// - /// Currently there is a 1:1 mapping between tracks and sources (source sharing is not available), - /// therefore this is equivalent to . - /// - public void RemoveLocalVideoTracksFromSource(ExternalVideoTrackSource source) - { - ThrowIfConnectionNotOpen(); - PeerConnectionInterop.PeerConnection_RemoveLocalVideoTracksFromSource(_nativePeerhandle, source._nativeHandle); - source.OnTracksRemovedFromSource(this); - } - - /// - /// Enable or disable the local audio track associated with this peer connection. - /// Disable audio tracks are still active, but are silent. - /// - /// true to enable the track, or false to disable it - /// The peer connection is not initialized. - public void SetLocalAudioTrackEnabled(bool enabled = true) - { - ThrowIfConnectionNotOpen(); - uint res = PeerConnectionInterop.PeerConnection_SetLocalAudioTrackEnabled(_nativePeerhandle, enabled ? -1 : 0); - Utils.ThrowOnErrorCode(res); - } - - /// - /// Check if the local audio track associated with this peer connection is enabled. - /// Disable audio tracks are still active, but are silent. - /// - /// true if the track is enabled, or false otherwise - /// The peer connection is not initialized. - public bool IsLocalAudioTrackEnabled() - { - ThrowIfConnectionNotOpen(); - return (PeerConnectionInterop.PeerConnection_IsLocalAudioTrackEnabled(_nativePeerhandle) != 0); - } + // Create the transceiver implementation + settings = settings ?? new TransceiverInitSettings(); + TransceiverInterop.InitConfig config = new TransceiverInterop.InitConfig(mediaKind, settings); + uint res = PeerConnectionInterop.PeerConnection_AddTransceiver(_nativePeerhandle, in config, out IntPtr transceiverHandle); + Utils.ThrowOnErrorCode(res); - /// - /// Remove from the current connection the local audio track added with . - /// - /// The peer connection is not initialized. - public void RemoveLocalAudioTrack() - { - ThrowIfConnectionNotOpen(); - PeerConnectionInterop.PeerConnection_RemoveLocalAudioTrack(_nativePeerhandle); + // The implementation fires the TransceiverAdded event, which creates the wrapper and + // stores a reference in the UserData of the native object. + IntPtr transceiver = TransceiverInterop.Transceiver_GetUserData(transceiverHandle); + Debug.Assert(transceiver != IntPtr.Zero); + var wrapper = Utils.ToWrapper(transceiver); + return wrapper; + } } #endregion - #region Data tracks + #region Data channels /// /// Add a new out-of-band data channel with the given ID. /// - /// A data channel is negotiated out-of-band when the peers agree on an identifier by any mean + /// A data channel is branded out-of-band when the peers agree on an identifier by any mean /// not known to WebRTC, and both open a data channel with that ID. The WebRTC will match the /// incoming and outgoing pipes by this ID to allow sending and receiving through that channel. /// /// This requires some external mechanism to agree on an available identifier not otherwise taken /// by another channel, and also requires to ensure that both peers explicitly open that channel. + /// The advantage of in-band data channels is that no SDP session renegotiation is needed, except + /// for the very first data channel added (in-band or out-of-band) which requires a negotiation + /// for the SCTP handshake (see remarks). /// /// The unique data channel identifier to use. /// The data channel name. @@ -1107,14 +1218,14 @@ public async Task AddDataChannelAsync(ushort id, string label, bool /// /// Add a new in-band data channel whose ID will be determined by the implementation. /// - /// A data channel is negotiated in-band when one peer requests its creation to the WebRTC core, + /// A data channel is branded in-band when one peer requests its creation to the WebRTC core, /// and the implementation negotiates with the remote peer an appropriate ID by sending some /// SDP offer message. In that case once accepted the other peer will automatically create the - /// appropriate data channel on its side with that negotiated ID, and the ID will be returned on + /// appropriate data channel on its side with that same ID, and the ID will be returned on /// both sides to the user for information. /// /// Compared to out-of-band messages, this requires exchanging some SDP messages, but avoids having - /// to determine a common unused ID and having to explicitly open the data channel on both sides. + /// to agree on a common unused ID and having to explicitly open the data channel on both sides. /// /// The data channel name. /// Indicates whether data channel messages are ordered (see @@ -1122,9 +1233,9 @@ public async Task AddDataChannelAsync(ushort id, string label, bool /// Indicates whether data channel messages are reliably delivered /// (see ). /// Returns a task which completes once the data channel is created. - /// The peer connection is not initialized. + /// The peer connection is not initialized. /// SCTP not negotiated. Call first. - /// Invalid data channel ID, must be in [0:65535]. + /// Invalid data channel ID, must be in [0:65535]. /// /// See the critical remark about SCTP handshake in . /// @@ -1143,53 +1254,65 @@ public async Task AddDataChannelAsync(string label, bool ordered, b /// Indicates whether data channel messages are reliably delivered /// (see ). /// Returns a task which completes once the data channel is created. - /// The peer connection is not initialized. - /// SCTP not negotiated. - /// Invalid data channel ID, must be in [0:65535]. + /// The peer connection is not initialized. + /// SCTP not negotiated. + /// Invalid data channel ID, must be in [0:65535]. private async Task AddDataChannelAsyncImpl(int id, string label, bool ordered, bool reliable) { ThrowIfConnectionNotOpen(); - // Create the native channel + // Create the native data channel return await Task.Run(() => { - // Create the wrapper + // Create the native object var config = new DataChannelInterop.CreateConfig { id = id, + flags = (ordered ? 0x1u : 0x0u) | (reliable ? 0x2u : 0x0u), label = label, - flags = (ordered ? 0x1u : 0x0u) | (reliable ? 0x2u : 0x0u) }; - DataChannelInterop.Callbacks callbacks; - var dataChannel = DataChannelInterop.CreateWrapper(this, config, out callbacks); - if (dataChannel == null) - { - return null; - } IntPtr nativeHandle = IntPtr.Zero; - var wrapperGCHandle = GCHandle.Alloc(dataChannel, GCHandleType.Normal); - var wrapperHandle = GCHandle.ToIntPtr(wrapperGCHandle); - uint res = PeerConnectionInterop.PeerConnection_AddDataChannel(_nativePeerhandle, wrapperHandle, config, callbacks, ref nativeHandle); - if (res == Utils.MRS_SUCCESS) - { - DataChannelInterop.SetHandle(dataChannel, nativeHandle); - return dataChannel; - } - - // Some error occurred, callbacks are not registered, so remove the GC lock. - wrapperGCHandle.Free(); - dataChannel.Dispose(); - dataChannel = null; - + uint res = PeerConnectionInterop.PeerConnection_AddDataChannel(_nativePeerhandle, ref config, ref nativeHandle); Utils.ThrowOnErrorCode(res); - return null; // for the compiler + + // The wrapper is created by the "DataChannelAdded" event invoked by PeerConnection_AddDataChannel(). + // Find it via the UserData property. + var dataChannelRef = DataChannelInterop.DataChannel_GetUserData(nativeHandle); + var dataChannelWrapper = Utils.ToWrapper(dataChannelRef); + Debug.Assert(dataChannelWrapper != null); + DataChannels.Add(dataChannelWrapper); + DataChannelAdded?.Invoke(dataChannelWrapper); + return dataChannelWrapper; }); } - internal bool RemoveDataChannel(IntPtr dataChannelHandle) + /// + /// Remove an existing data channel from the peer connection and destroy its native implementation. + /// + /// The data channel to remove and destroy. + /// The data channel is not owned by this peer connection. + public void RemoveDataChannel(DataChannel dataChannel) { ThrowIfConnectionNotOpen(); - return (PeerConnectionInterop.PeerConnection_RemoveDataChannel(_nativePeerhandle, dataChannelHandle) == Utils.MRS_SUCCESS); + if (DataChannels.Remove(dataChannel)) + { + // Notify the data channel is being closed. + // DestroyNative() will further change the state to Closed() once done. + dataChannel.State = DataChannel.ChannelState.Closing; + + var res = PeerConnectionInterop.PeerConnection_RemoveDataChannel(_nativePeerhandle, dataChannel._nativeHandle); + Utils.ThrowOnErrorCode(res); + + DataChannelRemoved?.Invoke(dataChannel); + + // PeerConnection is owning the data channel, and all internal states have been + // updated and events triggered, so notify the data channel to clean its internal state. + dataChannel.DestroyNative(); + } + else + { + throw new ArgumentException($"Data channel {dataChannel.Label} is not owned by peer connection {Name}.", "dataChannel"); + } } #endregion @@ -1234,6 +1357,8 @@ public bool CreateOffer() /// Create an SDP answer message to a previously-received offer, to accept a connection. /// Once the message is ready to be sent, the event is fired /// to allow the user to send that message to the remote peer via its selected signaling solution. + /// Note that this cannot be called before + /// successfully completed and applied the remote offer. /// /// true if the answer creation task was successfully submitted. /// The peer connection is not initialized. @@ -1681,9 +1806,9 @@ private void ThrowIfConnectionNotOpen() /// /// The list of available video capture devices. /// - /// Assign one of the returned to the - /// field to force a local video - /// track to use that device when creating it with . + /// Assign one of the returned to the + /// field to force a local video track to use that device when creating it with + /// . /// public static Task> GetVideoCaptureDevicesAsync() { @@ -1853,12 +1978,14 @@ internal void OnConnected() internal void OnDataChannelAdded(DataChannel dataChannel) { MainEventSource.Log.DataChannelAdded(dataChannel.ID, dataChannel.Label); + DataChannels.Add(dataChannel); DataChannelAdded?.Invoke(dataChannel); } internal void OnDataChannelRemoved(DataChannel dataChannel) { MainEventSource.Log.DataChannelRemoved(dataChannel.ID, dataChannel.Label); + DataChannels.Remove(dataChannel); DataChannelRemoved?.Invoke(dataChannel); } @@ -1944,43 +2071,145 @@ internal void OnIceGatheringStateChanged(IceGatheringState newState) internal void OnRenegotiationNeeded() { MainEventSource.Log.RenegotiationNeeded(); - RenegotiationNeeded?.Invoke(); + // Check if the RenegotiationNeeded event is temporarily suspended. This happens while + // creating a new Transceiver, until the wrapper has finished syncing. + // The current callback is invoked from the signaling thread by the WebRTC implementation. + // It should free the thread ASAP and not re-enter it with another API call. Since + // typically users will call CreateOffer() in response to this event, defer the event + // handling to a .NET worker thread instead and return immediately to the WebRTC signaling one. + _renegotiationNeededEvent.InvokeAsync(); } - internal void OnTrackAdded(TrackKind trackKind) + /// + /// Insert the given transceiver into the list at the index + /// corresponding to its . + /// + /// Transceiver object to insert. + internal void InsertTransceiverNoLock(Transceiver transceiver) + { + int mlineIndex = transceiver.MlineIndex; + if (mlineIndex >= Transceivers.Count) + { + while (mlineIndex >= Transceivers.Count + 1) + { + Transceivers.Add(null); + } + Transceivers.Add(transceiver); + } + else + { + Debug.Assert(Transceivers[mlineIndex] == null); + Transceivers[mlineIndex] = transceiver; + } + } + + /// + /// Callback on transceiver created for the peer connection, irrelevant of whether + /// it has tracks or not. This is called both when created from the managed side or + /// from the native side. + /// + /// The newly created transceiver which has this peer connection as owner + internal void OnTransceiverAdded(Transceiver tr) { - MainEventSource.Log.TrackAdded(trackKind); - TrackAdded?.Invoke(trackKind); + lock (_tracksLock) + { + InsertTransceiverNoLock(tr); + } + TransceiverAdded?.Invoke(tr); } - internal void OnTrackRemoved(TrackKind trackKind) + /// + /// Callback invoked by the native implementation when a transceiver starts receiving, + /// and a remote audio track is created as a result to receive its audio data. + /// + /// The newly created remote audio track. + /// The audio transceiver now receiving from the remote peer. + internal void OnAudioTrackAdded(RemoteAudioTrack track, Transceiver transceiver) { - MainEventSource.Log.TrackRemoved(trackKind); - TrackRemoved?.Invoke(trackKind); + MainEventSource.Log.AudioTrackAdded(track.Name); + + Debug.Assert(transceiver.MediaKind == MediaKind.Audio); + Debug.Assert(track.Transceiver == null); + Debug.Assert(transceiver.RemoteAudioTrack == null); + // track.PeerConnection was set in its constructor + track.Transceiver = transceiver; + transceiver.RemoteAudioTrack = track; + + AudioTrackAdded?.Invoke(track); } - internal void OnI420ARemoteVideoFrameReady(I420AVideoFrame frame) + /// + /// Callback invoked by the native implementation when a transceiver stops receiving, + /// and a remote audio track is removed from it as a result. + /// + /// The remote audio track removed from the audio transceiver. + internal void OnAudioTrackRemoved(RemoteAudioTrack track) { - MainEventSource.Log.I420ARemoteVideoFrameReady(frame.width, frame.height); - I420ARemoteVideoFrameReady?.Invoke(frame); + MainEventSource.Log.AudioTrackRemoved(track.Name); + Transceiver transceiver = track.Transceiver; // cache before removed + + Debug.Assert(track.PeerConnection == this); + Debug.Assert(transceiver.RemoteAudioTrack == track); + Debug.Assert(track.Transceiver == transceiver); + track.PeerConnection = null; + transceiver.RemoteAudioTrack = null; + track.Transceiver = null; + + AudioTrackRemoved?.Invoke(transceiver, track); + + // PeerConnection is owning the remote track, and all internal states have been + // updated and events triggered, so notify the track to clean its internal state. + track.DestroyNative(); } - internal void OnArgb32RemoteVideoFrameReady(Argb32VideoFrame frame) + /// + /// Callback invoked by the native implementation when a transceiver starts receiving, + /// and a remote video track is created as a result to receive its video data. + /// + /// The newly created remote video track. + /// The video transceiver now receiving from the remote peer. + internal void OnVideoTrackAdded(RemoteVideoTrack track, Transceiver transceiver) { - MainEventSource.Log.Argb32RemoteVideoFrameReady(frame.width, frame.height); - Argb32RemoteVideoFrameReady?.Invoke(frame); + MainEventSource.Log.VideoTrackAdded(track.Name); + + Debug.Assert(transceiver.MediaKind == MediaKind.Video); + Debug.Assert(track.Transceiver == null); + Debug.Assert(transceiver.RemoteVideoTrack == null); + // track.PeerConnection was set in its constructor + track.Transceiver = transceiver; + transceiver.RemoteVideoTrack = track; + + VideoTrackAdded?.Invoke(track); } - internal void OnLocalAudioFrameReady(AudioFrame frame) + /// + /// Callback invoked by the native implementation when a transceiver stops receiving, + /// and a remote video track is removed from it as a result. + /// + /// The remote video track removed from the video transceiver. + internal void OnVideoTrackRemoved(RemoteVideoTrack track) { - MainEventSource.Log.LocalAudioFrameReady(frame.bitsPerSample, frame.channelCount, frame.sampleCount); - LocalAudioFrameReady?.Invoke(frame); + MainEventSource.Log.VideoTrackRemoved(track.Name); + Transceiver transceiver = track.Transceiver; // cache before removed + + Debug.Assert(track.PeerConnection == this); + Debug.Assert(transceiver.RemoteVideoTrack == track); + Debug.Assert(track.Transceiver == transceiver); + track.PeerConnection = null; + transceiver.RemoteVideoTrack = null; + track.Transceiver = null; + + VideoTrackRemoved?.Invoke(transceiver, track); + + // PeerConnection is owning the remote track, and all internal states have been + // updated and events triggered, so notify the track to clean its internal state. + track.DestroyNative(); } - internal void OnRemoteAudioFrameReady(AudioFrame frame) + /// + public override string ToString() { - MainEventSource.Log.RemoteAudioFrameReady(frame.bitsPerSample, frame.channelCount, frame.sampleCount); - RemoteAudioFrameReady?.Invoke(frame); + return $"(PeerConnection)\"{Name}\""; } } } diff --git a/libs/Microsoft.MixedReality.WebRTC/RemoteAudioTrack.cs b/libs/Microsoft.MixedReality.WebRTC/RemoteAudioTrack.cs new file mode 100644 index 000000000..d93dba996 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/RemoteAudioTrack.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using Microsoft.MixedReality.WebRTC.Interop; +using Microsoft.MixedReality.WebRTC.Tracing; + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Audio track receiving audio frames from the remote peer. + /// + public class RemoteAudioTrack : MediaTrack + { + /// + /// Enabled status of the track. If enabled, receives audio frames from the remote peer as + /// expected. If disabled, does not receive anything (silence). + /// + /// + /// Reading the value of this property after the track has been disposed is valid, and returns + /// false. + /// The remote audio track enabled status is controlled by the remote peer only. + /// + public bool Enabled + { + get + { + return (RemoteAudioTrackInterop.RemoteAudioTrack_IsEnabled(_nativeHandle) != 0); + } + } + + /// + /// Event that occurs when a audio frame has been received from the remote peer. + /// + public event AudioFrameDelegate AudioFrameReady; + + /// + /// Handle to the native RemoteAudioTrack object. + /// + /// + /// In native land this is a Microsoft::MixedReality::WebRTC::RemoteAudioTrackHandle. + /// + internal IntPtr _nativeHandle = IntPtr.Zero; + + /// + /// Handle to self for interop callbacks. This adds a reference to the current object, preventing + /// it from being garbage-collected. + /// + private IntPtr _selfHandle = IntPtr.Zero; + + /// + /// Callback arguments to ensure delegates registered with the native layer don't go out of scope. + /// + private RemoteAudioTrackInterop.InteropCallbackArgs _interopCallbackArgs; + + // Constructor for interop-based creation; SetHandle() will be called later + internal RemoteAudioTrack(IntPtr handle, PeerConnection peer, string trackName) : base(peer, trackName) + { + _nativeHandle = handle; + RegisterInteropCallbacks(); + } + + private void RegisterInteropCallbacks() + { + _interopCallbackArgs = new RemoteAudioTrackInterop.InteropCallbackArgs() + { + Track = this, + FrameCallback = RemoteAudioTrackInterop.FrameCallback, + }; + _selfHandle = Utils.MakeWrapperRef(this); + RemoteAudioTrackInterop.RemoteAudioTrack_RegisterFrameCallback( + _nativeHandle, _interopCallbackArgs.FrameCallback, _selfHandle); + } + + private void UnregisterInteropCallbacks() + { + if (_selfHandle != IntPtr.Zero) + { + RemoteAudioTrackInterop.RemoteAudioTrack_RegisterFrameCallback(_nativeHandle, null, IntPtr.Zero); + Utils.ReleaseWrapperRef(_selfHandle); + _selfHandle = IntPtr.Zero; + _interopCallbackArgs = null; + } + } + + /// + /// Dispose of the native track. Invoked by its owner (). + /// + internal void DestroyNative() + { + if (_nativeHandle == IntPtr.Zero) + { + return; + } + + Debug.Assert(PeerConnection == null); // see OnTrackRemoved + + UnregisterInteropCallbacks(); + + _nativeHandle = IntPtr.Zero; + } + + internal void OnFrameReady(AudioFrame frame) + { + MainEventSource.Log.RemoteAudioFrameReady(frame.bitsPerSample, frame.sampleRate, frame.channelCount, frame.sampleCount); + AudioFrameReady?.Invoke(frame); + } + + internal override void OnMute(bool muted) + { + + } + + /// + public override string ToString() + { + return $"(RemoteAudioTrack)\"{Name}\""; + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/RemoteVideoTrack.cs b/libs/Microsoft.MixedReality.WebRTC/RemoteVideoTrack.cs new file mode 100644 index 000000000..5036a766c --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/RemoteVideoTrack.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using Microsoft.MixedReality.WebRTC.Interop; +using Microsoft.MixedReality.WebRTC.Tracing; + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Video track receiving video frames from the remote peer. + /// + public class RemoteVideoTrack : MediaTrack + { + /// + /// Enabled status of the track. If enabled, receives video frames from the remote peer as + /// expected. If disabled, receives only black frames instead. + /// + /// + /// Reading the value of this property after the track has been disposed is valid, and returns + /// false. + /// The remote video track enabled status is controlled by the remote peer only. + /// + public bool Enabled + { + get + { + return (RemoteVideoTrackInterop.RemoteVideoTrack_IsEnabled(_nativeHandle) != 0); + } + } + + /// + /// Event that occurs when a video frame has been received from the remote peer. + /// + public event I420AVideoFrameDelegate I420AVideoFrameReady; + + /// + /// Event that occurs when a video frame has been received from the remote peer. + /// + public event Argb32VideoFrameDelegate Argb32VideoFrameReady; + + /// + /// Handle to the native RemoteVideoTrack object. + /// + /// + /// In native land this is a Microsoft::MixedReality::WebRTC::RemoteVideoTrackHandle. + /// + internal IntPtr _nativeHandle = IntPtr.Zero; + + /// + /// Handle to self for interop callbacks. This adds a reference to the current object, preventing + /// it from being garbage-collected. + /// + private IntPtr _selfHandle = IntPtr.Zero; + + /// + /// Callback arguments to ensure delegates registered with the native layer don't go out of scope. + /// + private RemoteVideoTrackInterop.InteropCallbackArgs _interopCallbackArgs; + + // Constructor for interop-based creation; SetHandle() will be called later + internal RemoteVideoTrack(IntPtr handle, PeerConnection peer, string trackName) : base(peer, trackName) + { + _nativeHandle = handle; + RegisterInteropCallbacks(); + } + + private void RegisterInteropCallbacks() + { + _interopCallbackArgs = new RemoteVideoTrackInterop.InteropCallbackArgs() + { + Track = this, + I420AFrameCallback = RemoteVideoTrackInterop.I420AFrameCallback, + Argb32FrameCallback = RemoteVideoTrackInterop.Argb32FrameCallback, + }; + _selfHandle = Utils.MakeWrapperRef(this); + RemoteVideoTrackInterop.RemoteVideoTrack_RegisterI420AFrameCallback( + _nativeHandle, _interopCallbackArgs.I420AFrameCallback, _selfHandle); + RemoteVideoTrackInterop.RemoteVideoTrack_RegisterArgb32FrameCallback( + _nativeHandle, _interopCallbackArgs.Argb32FrameCallback, _selfHandle); + } + + private void UnregisterInteropCallbacks() + { + if (_selfHandle != IntPtr.Zero) + { + RemoteVideoTrackInterop.RemoteVideoTrack_RegisterI420AFrameCallback(_nativeHandle, null, IntPtr.Zero); + RemoteVideoTrackInterop.RemoteVideoTrack_RegisterArgb32FrameCallback(_nativeHandle, null, IntPtr.Zero); + Utils.ReleaseWrapperRef(_selfHandle); + _selfHandle = IntPtr.Zero; + _interopCallbackArgs = null; + } + } + + /// + /// Dispose of the native track. Invoked by its owner (). + /// + internal void DestroyNative() + { + if (_nativeHandle == IntPtr.Zero) + { + return; + } + + Debug.Assert(PeerConnection == null); // see OnTrackRemoved + + UnregisterInteropCallbacks(); + + _nativeHandle = IntPtr.Zero; + } + + internal void OnI420AFrameReady(I420AVideoFrame frame) + { + MainEventSource.Log.I420ARemoteVideoFrameReady(frame.width, frame.height); + I420AVideoFrameReady?.Invoke(frame); + } + + internal void OnArgb32FrameReady(Argb32VideoFrame frame) + { + MainEventSource.Log.Argb32RemoteVideoFrameReady(frame.width, frame.height); + Argb32VideoFrameReady?.Invoke(frame); + } + + internal override void OnMute(bool muted) + { + + } + + /// + public override string ToString() + { + return $"(RemoteVideoTrack)\"{Name}\""; + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Tracing/MainEventSource.cs b/libs/Microsoft.MixedReality.WebRTC/Tracing/MainEventSource.cs index 10cc9793e..6226e1a0e 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Tracing/MainEventSource.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Tracing/MainEventSource.cs @@ -122,13 +122,10 @@ public void AddIceCandidate(string sdpMid, int sdpMlineindex, string candidate) #endregion - #region Media - - [Event(0x3001, Level = EventLevel.Informational, Keywords = Keywords.Media)] - public void TrackAdded(PeerConnection.TrackKind trackKind) { WriteEvent(0x3001, (int)trackKind); } - - [Event(0x3002, Level = EventLevel.Informational, Keywords = Keywords.Media)] - public void TrackRemoved(PeerConnection.TrackKind trackKind) { WriteEvent(0x3002, (int)trackKind); } + #region Media + + // 0x3001 - TrackAdded (deprecated) + // 0x3002 - TrackRemoved (deprecated) [Event(0x3003, Level = EventLevel.Verbose, Keywords = Keywords.Media)] public void I420ALocalVideoFrameReady(uint width, uint height) { WriteEvent(0x3003, (int)width, (int)height); } @@ -143,15 +140,15 @@ public void AddIceCandidate(string sdpMid, int sdpMlineindex, string candidate) public void Argb32RemoteVideoFrameReady(uint width, uint height) { WriteEvent(0x3006, (int)width, (int)height); } [Event(0x3007, Level = EventLevel.Verbose, Keywords = Keywords.Media)] - public void LocalAudioFrameReady(uint bitsPerSample, uint channelCount, uint frameCount) + public void LocalAudioFrameReady(uint bitsPerSample, uint sampleRate, uint channelCount, uint sampleCount) { - WriteEvent(0x3007, (int)bitsPerSample, (int)channelCount, (int)frameCount); + WriteEvent(0x3007, (int)bitsPerSample, (int)sampleRate, (int)channelCount, (int)sampleCount); } [Event(0x3008, Level = EventLevel.Verbose, Keywords = Keywords.Media)] - public void RemoteAudioFrameReady(uint bitsPerSample, uint channelCount, uint frameCount) + public void RemoteAudioFrameReady(uint bitsPerSample, uint sampleRate, uint channelCount, uint sampleCount) { - WriteEvent(0x3008, (int)bitsPerSample, (int)channelCount, (int)frameCount); + WriteEvent(0x3008, (int)bitsPerSample, (int)sampleRate, (int)channelCount, (int)sampleCount); } [Event(0x3100, Level = EventLevel.Verbose, Keywords = Keywords.Media)] @@ -202,6 +199,18 @@ public unsafe void VideoFrameQueueTrackLateFrame(Guid instanceId, float queuedDt WriteEvent(0x3107, instanceId, queuedDtMs, dequeuedDtMs); } + [Event(0x3108, Level = EventLevel.Informational, Keywords = Keywords.Media)] + public void VideoTrackAdded(string trackName) { WriteEvent(0x3108, trackName); } + + [Event(0x3109, Level = EventLevel.Informational, Keywords = Keywords.Media)] + public void VideoTrackRemoved(string trackName) { WriteEvent(0x3109, trackName); } + + [Event(0x3208, Level = EventLevel.Informational, Keywords = Keywords.Media)] + public void AudioTrackAdded(string trackName) { WriteEvent(0x3208, trackName); } + + [Event(0x3209, Level = EventLevel.Informational, Keywords = Keywords.Media)] + public void AudioTrackRemoved(string trackName) { WriteEvent(0x3209, trackName); } + #endregion #region DataChannel @@ -231,6 +240,21 @@ public void DataChannelBufferingChanged(int id, ulong previous, ulong current, u #region EventWrite overloads + [NonEvent] + public unsafe void WriteEvent(int eventId, int arg1, int arg2, int arg3, int arg4) + { + EventData* dataDesc = stackalloc EventData[4]; + dataDesc[0].DataPointer = (IntPtr)(&arg1); + dataDesc[0].Size = 4; + dataDesc[1].DataPointer = (IntPtr)(&arg2); + dataDesc[1].Size = 4; + dataDesc[2].DataPointer = (IntPtr)(&arg3); + dataDesc[2].Size = 4; + dataDesc[3].DataPointer = (IntPtr)(&arg4); + dataDesc[3].Size = 4; + WriteEventCore(eventId, 4, dataDesc); + } + [NonEvent] public unsafe void WriteEvent(int eventId, string arg1, int arg2, string arg3) { diff --git a/libs/Microsoft.MixedReality.WebRTC/Transceiver.cs b/libs/Microsoft.MixedReality.WebRTC/Transceiver.cs new file mode 100644 index 000000000..2c468559a --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Transceiver.cs @@ -0,0 +1,452 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using Microsoft.MixedReality.WebRTC.Interop; + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Type of media track or media transceiver. + /// + /// + /// This is the projection of mrsMediaKind from the interop API. + /// + public enum MediaKind : uint + { + /// + /// Audio data. + /// + Audio = 0, + + /// + /// Video data. + /// + Video = 1 + } + + /// + /// Transceiver of a peer connection. + /// + /// A transceiver is a media "pipe" connecting the local and remote peers, and used to transmit media + /// data (audio or video) between the peers. The transceiver has a media flow direction indicating whether + /// it is sending and/or receiving any media, or is inactive. When sending some media, the transceiver's + /// local track is used as the source of that media. Conversely, when receiving some media, that media is + /// delivered to the remote media track of the transceiver. As a convenience, both tracks can be null to + /// avoid sending or ignore the received media, although this does not influence the media flow direction. + /// + /// Transceivers are owned by the peer connection which creates them, and cannot be destroyed nor removed + /// from the peer connection. They become invalid when the peer connection is closed, and should not be + /// used after that. + /// + /// + /// This object corresponds roughly to the same-named notion in the WebRTC standard when using the + /// Unified Plan SDP semantic. + /// For Plan B, where RTP transceivers are not available, this wrapper tries to emulate the Unified Plan + /// transceiver concept, and is therefore providing an abstraction over the WebRTC concept of transceivers. + /// + /// + public class Transceiver + { + /// + /// Direction of the media flowing inside the transceiver. + /// + public enum Direction : int + { + /// + /// Transceiver is both sending to and receiving from the remote peer connection. + /// + SendReceive = 0, + + /// + /// Transceiver is sending to the remote peer, but is not receiving any media from the remote peer. + /// + SendOnly = 1, + + /// + /// Transceiver is receiving from the remote peer, but is not sending any media to the remote peer. + /// + ReceiveOnly = 2, + + /// + /// Transceiver is inactive, neither sending nor receiving any media data. + /// + Inactive = 3, + } + + /// + /// A name for the transceiver, used for logging and debugging only. + /// This can be set on construction if the transceiver is created by the local peer using + /// , or will + /// be generated by the implementation otherwise. + /// There is no guarantee of unicity; this name is only informational. + /// + public string Name { get; } = string.Empty; + + /// + /// Type of media carried by the transceiver, and by extension type of media of its tracks. + /// + public MediaKind MediaKind { get; } + + /// + /// Peer connection this transceiver is part of. + /// + /// + public PeerConnection PeerConnection { get; } = null; + + /// + /// Index of the media line in the SDP protocol for this transceiver. This also corresponds + /// to the index of the transceiver inside . + /// + /// + /// For Plan B, the media line index is not guaranteed by the SDP protocol, but this index + /// is still valid as the index of the transceiver in the global collection of the peer connection + /// . + /// + public int MlineIndex { get; } = -1; + + /// + /// Transceiver direction desired by the user. + /// Once changed by the user, this value is the next direction that will be negotiated when + /// calling or . + /// After the negotiation is completed, this is equal to . + /// Setting this value triggers a event. + /// + /// + public Direction DesiredDirection + { + get { return _desiredDirection; } + set + { + if (value == _desiredDirection) + { + return; + } + var res = TransceiverInterop.Transceiver_SetDirection(_nativeHandle, value); + Utils.ThrowOnErrorCode(res); + _desiredDirection = value; + } + } + + /// + /// Last negotiated transceiver direction. This is equal to + /// after a negotiation is completed, but remains constant when changing + /// until the next SDP negotiation. + /// + /// + public Direction? NegotiatedDirection { get; protected set; } = null; + + /// + /// List of stream IDs associated with the transceiver. + /// + public string[] StreamIDs { get; } + + /// + /// Local track attached to the transceiver, which is used to send data to the remote peer + /// if includes sending. + /// This cannot be assigned directly; instead use or + /// depending on the media kind of the transceiver. + /// + /// + /// + public MediaTrack LocalTrack => _localTrack; + + /// + /// Local audio track attached to the transceiver, if is . + /// The property has two uses: + /// - as a convenience getter to retrieve already cast to a type. + /// - to attach a new local audio track if the transceiver is an audio transceiver; otherwise this throws a . + /// + public LocalAudioTrack LocalAudioTrack + { + get { return (_localTrack as LocalAudioTrack); } + set + { + if (MediaKind == MediaKind.Audio) + { + SetLocalTrackImpl(value); + } + else + { + throw new ArgumentException("Cannot assign local audio track as local track of video transceiver"); + } + } + } + + /// + /// Local video track attached to the transceiver, if is . + /// The property has two uses: + /// - as a convenience getter to retrieve already cast to a type. + /// - to attach a new local video track if the transceiver is a video transceiver; otherwise this throws a . + /// + public LocalVideoTrack LocalVideoTrack + { + get { return (_localTrack as LocalVideoTrack); } + set + { + if (MediaKind == MediaKind.Video) + { + SetLocalTrackImpl(value); + } + else + { + throw new ArgumentException("Cannot assign local video track as local track of audio transceiver"); + } + } + } + + /// + /// Remote track attached to the transceiver, which is used to receive data from the + /// remote peer if includes receiving. + /// This cannot be assigned. This is updated automatically when the remote track is + /// created or destroyed as part of a renegotiation. + /// + /// + /// + public MediaTrack RemoteTrack => _remoteTrack; + + /// + /// Remote audio track attached to the transceiver, if is . + /// This is equivalent to for audio transceivers, and null otherwise. + /// + public RemoteAudioTrack RemoteAudioTrack + { + get { return (_remoteTrack as RemoteAudioTrack); } + internal set { _remoteTrack = value; } + } + + /// + /// Remote video track attached to the transceiver, if is . + /// This is equivalent to for video transceivers, and null otherwise. + /// + public RemoteVideoTrack RemoteVideoTrack + { + get { return (_remoteTrack as RemoteVideoTrack); } + internal set { _remoteTrack = value; } + } + + /// + /// Backing field for . + /// + /// + protected Direction _desiredDirection; + + /// + /// Handle to the native transceiver object, valid until + /// is called by the native implementation when the transceiver is destroyed as part + /// of the peer connection closing. + /// + /// + /// In native land this is a mrsTransceiverHandle. + /// + internal IntPtr _nativeHandle = IntPtr.Zero; + + /// + /// Reference to the struct keeping the callback delegates alive while registered with + /// the native implementation. + /// This should be released with . + /// + /// + private IntPtr _argsRef = IntPtr.Zero; + + private MediaTrack _localTrack = null; + private MediaTrack _remoteTrack = null; + + /// + /// Create a new transceiver associated with a given peer connection. + /// + /// Handle to the native transceiver object. + /// The media kind of the transceiver and its tracks. + /// The peer connection owning this transceiver. + /// The transceiver media line index in SDP. + /// The transceiver name. + /// Collection of stream IDs the transceiver is associated with, as set by the peer which created it. + /// Initial value to initialize with. + internal Transceiver(IntPtr handle, MediaKind mediaKind, PeerConnection peerConnection, int mlineIndex, + string name, string[] streamIDs, Direction initialDesiredDirection) + { + Debug.Assert(handle != IntPtr.Zero); + _nativeHandle = handle; + MediaKind = mediaKind; + PeerConnection = peerConnection; + MlineIndex = mlineIndex; + Name = name; + StreamIDs = streamIDs; + _desiredDirection = initialDesiredDirection; + TransceiverInterop.RegisterCallbacks(this, out _argsRef); + } + + /// + /// Change the local audio track sending data to the remote peer. + /// + /// This detaches the previous local audio track if any, and attaches the new one instead. + /// Note that the transceiver will only send some audio data to the remote peer if its + /// negotiated direction includes sending some data and it has an attached local track to + /// produce this data. + /// + /// This change is transparent to the session, and does not trigger any renegotiation. + /// + /// The new local audio track attached to the transceiver, and used to + /// produce audio data to send to the remote peer if the transceiver is sending. + /// Passing null is allowed, and will detach the current track if any. + private void SetLocalTrackImpl(MediaTrack track) + { + if (track == _localTrack) + { + return; + } + + var audioTrack = (track as LocalAudioTrack); + var videoTrack = (track as LocalVideoTrack); + if ((audioTrack != null) && (MediaKind != MediaKind.Audio)) + { + throw new ArgumentException("Cannot set local audio track as local track of video transceiver"); + } + if ((videoTrack != null) && (MediaKind != MediaKind.Video)) + { + throw new ArgumentException("Cannot set local video track as local track of audio transceiver"); + } + + if (track != null) + { + if ((track.PeerConnection != null) && (track.PeerConnection != PeerConnection)) + { + throw new InvalidOperationException($"Cannot set track {track} of peer connection {track.PeerConnection} on transceiver {this} of different peer connection {PeerConnection}."); + } + uint res = Utils.MRS_E_UNKNOWN; + if (audioTrack != null) + { + res = TransceiverInterop.Transceiver_SetLocalAudioTrack(_nativeHandle, audioTrack._nativeHandle); + } + else if (videoTrack != null) + { + res = TransceiverInterop.Transceiver_SetLocalVideoTrack(_nativeHandle, videoTrack._nativeHandle); + } + Utils.ThrowOnErrorCode(res); + } + else + { + // Note: Cannot pass null for SafeHandle parameter value (ArgumentNullException) + uint res = Utils.MRS_E_UNKNOWN; + if (MediaKind == MediaKind.Audio) + { + res = TransceiverInterop.Transceiver_SetLocalAudioTrack(_nativeHandle, new LocalAudioTrackHandle()); + } + else if (MediaKind == MediaKind.Video) + { + res = TransceiverInterop.Transceiver_SetLocalVideoTrack(_nativeHandle, new LocalVideoTrackHandle()); + } + Utils.ThrowOnErrorCode(res); + } + + // Remove old track + if (_localTrack != null) + { + Debug.Assert(_localTrack.Transceiver == this); + Debug.Assert(_localTrack.PeerConnection == PeerConnection); + _localTrack.Transceiver = null; + _localTrack.PeerConnection = null; + _localTrack = null; + } + + // Add new track + if (track != null) + { + Debug.Assert(track.Transceiver == null); + Debug.Assert(track.PeerConnection == null); + _localTrack = track; + _localTrack.Transceiver = this; + _localTrack.PeerConnection = PeerConnection; + } + } + + /// + /// Callback invoked after the native transceiver has been destroyed, for clean-up. + /// This is called by the peer connection when it closes, just before the C# transceiver + /// object instance is destroyed. + /// This replaces an hypothetical TransceiverRemoved callback, which doesn't exist to + /// prevent confusion and underline the fact transceiver cannot be removed after being + /// added to a peer connection, until that peer connection is closed and destroys them. + /// + internal void CleanUpAfterNativeDestroyed() + { + Debug.Assert(_localTrack == null); + Debug.Assert(_remoteTrack == null); + Debug.Assert(_nativeHandle != IntPtr.Zero); + _nativeHandle = IntPtr.Zero; + // No need (and can't) unregister callbacks, the native transceiver is already destroyed + Utils.ReleaseWrapperRef(_argsRef); + _argsRef = IntPtr.Zero; + } + + /// + /// Callback on internal implementation state changed to synchronize the cached state of this wrapper. + /// + /// Current negotiated direction of the transceiver + /// Current desired direction of the transceiver + internal void OnStateUpdated(Direction? negotiatedDirection, Direction desiredDirection) + { + _desiredDirection = desiredDirection; + + if (negotiatedDirection != NegotiatedDirection) + { + bool hadSendBefore = HasSend(NegotiatedDirection); + bool hasSendNow = HasSend(negotiatedDirection); + bool hadRecvBefore = HasRecv(NegotiatedDirection); + bool hasRecvNow = HasRecv(negotiatedDirection); + + NegotiatedDirection = negotiatedDirection; + + if (hadSendBefore != hasSendNow) + { + _localTrack?.OnMute(!hasSendNow); + } + if (hadRecvBefore != hasRecvNow) + { + _remoteTrack?.OnMute(!hasRecvNow); + } + } + } + + /// + /// Check whether the given direction includes sending. + /// + /// The direction to check. + /// true if direction is or . + public static bool HasSend(Direction dir) + { + return (dir == Direction.SendOnly) || (dir == Direction.SendReceive); + } + + /// + /// Check whether the given direction includes receiving. + /// + /// The direction to check. + /// true if direction is or . + public static bool HasRecv(Direction dir) + { + return (dir == Direction.ReceiveOnly) || (dir == Direction.SendReceive); + } + + /// + /// Check whether the given direction includes sending. + /// + /// The direction to check. + /// true if direction is or . + public static bool HasSend(Direction? dir) + { + return dir.HasValue && ((dir == Direction.SendOnly) || (dir == Direction.SendReceive)); + } + + /// + /// Check whether the given direction includes receiving. + /// + /// The direction to check. + /// true if direction is or . + public static bool HasRecv(Direction? dir) + { + return dir.HasValue && ((dir == Direction.ReceiveOnly) || (dir == Direction.SendReceive)); + } + } +} diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/DataChannelTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/DataChannelTests.cs index 6de1071de..aa938e2f3 100644 --- a/tests/Microsoft.MixedReality.WebRTC.Tests/DataChannelTests.cs +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/DataChannelTests.cs @@ -10,70 +10,57 @@ namespace Microsoft.MixedReality.WebRTC.Tests { - [TestFixture] - internal class DataChannelTests + [TestFixture(SdpSemantic.PlanB)] + [TestFixture(SdpSemantic.UnifiedPlan)] + internal class DataChannelTests : PeerConnectionTestBase { + public DataChannelTests(SdpSemantic sdpSemantic) : base(sdpSemantic) + { + } + [Test] public async Task InBand() { - // Setup - var config = new PeerConnectionConfiguration(); - var pc1 = new PeerConnection(); - var pc2 = new PeerConnection(); - await pc1.InitializeAsync(config); - await pc2.InitializeAsync(config); - pc1.LocalSdpReadytoSend += async (string type, string sdp) => - { - await pc2.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - pc2.CreateAnswer(); - }; - pc2.LocalSdpReadytoSend += async (string type, string sdp) => - { - await pc1.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - pc1.CreateAnswer(); - }; - pc1.IceCandidateReadytoSend += (string candidate, int sdpMlineindex, string sdpMid) - => pc2.AddIceCandidate(sdpMid, sdpMlineindex, candidate); - pc2.IceCandidateReadytoSend += (string candidate, int sdpMlineindex, string sdpMid) - => pc1.AddIceCandidate(sdpMid, sdpMlineindex, candidate); + // Disable auto-renegotiation + suspendOffer1_ = true; + suspendOffer2_ = true; // Add dummy out-of-band data channel to force SCTP negotiating. // Otherwise after connecting AddDataChannelAsync() will fail. - await pc1.AddDataChannelAsync(42, "dummy", false, false); - await pc2.AddDataChannelAsync(42, "dummy", false, false); + await pc1_.AddDataChannelAsync(42, "dummy", false, false); + await pc2_.AddDataChannelAsync(42, "dummy", false, false); + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(renegotiationEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + renegotiationEvent1_.Reset(); + renegotiationEvent2_.Reset(); // Connect - { - var c1 = new ManualResetEventSlim(false); - var c2 = new ManualResetEventSlim(false); - pc1.Connected += () => c1.Set(); - pc2.Connected += () => c2.Set(); - Assert.True(pc1.CreateOffer()); - Assert.True(c1.Wait(TimeSpan.FromSeconds(60.0))); - Assert.True(c2.Wait(TimeSpan.FromSeconds(60.0))); - Assert.True(pc1.IsConnected); - Assert.True(pc1.IsConnected); - } + StartOfferWith(pc1_); + WaitForSdpExchangeCompleted(); + + // Ensure auto-renegotiation is active + suspendOffer1_ = false; + suspendOffer2_ = false; // Negotiate data channel in-band DataChannel data1 = null; DataChannel data2 = null; { var c2 = new ManualResetEventSlim(false); - pc2.DataChannelAdded += (DataChannel channel) => + pc2_.DataChannelAdded += (DataChannel channel) => { data2 = channel; c2.Set(); }; - data1 = await pc1.AddDataChannelAsync("test_data_channel", true, true); + // Note that for SCTP data channels (always the case in MixedReality-WebRTC) a renegotiation + // needed event is triggered only on the first data channel created. Since dummy channels were + // added above to trigger SCTP handshake, this one below will not trigger a renegotiation event. + data1 = await pc1_.AddDataChannelAsync("test_data_channel", ordered: true, reliable: true); Assert.IsNotNull(data1); Assert.True(c2.Wait(TimeSpan.FromSeconds(60.0))); Assert.IsNotNull(data2); + Assert.AreEqual(data1.ID, data2.ID); Assert.AreEqual(data1.Label, data2.Label); - // Do not test DataChannel.ID; at this point for in-band channels the ID has not - // been agreed upon with the remote peer yet. } // Send data @@ -90,62 +77,76 @@ public async Task InBand() data1.SendMessage(msg); Assert.True(c2.Wait(TimeSpan.FromSeconds(60.0))); } - - // Clean-up - pc1.Close(); - pc1.Dispose(); - pc2.Close(); - pc2.Dispose(); } [Test] - public async Task SctpError() + public async Task OutOfBand() { - // Setup - var config = new PeerConnectionConfiguration(); - var pc1 = new PeerConnection(); - var pc2 = new PeerConnection(); - await pc1.InitializeAsync(config); - await pc2.InitializeAsync(config); - pc1.LocalSdpReadytoSend += async (string type, string sdp) => + // Disable auto-renegotiation + suspendOffer1_ = true; + suspendOffer2_ = true; + + // Add out-of-band data channels + const int DataChannelId = 42; + DataChannel data1 = await pc1_.AddDataChannelAsync(DataChannelId, "my_channel", ordered: true, reliable: true); + DataChannel data2 = await pc2_.AddDataChannelAsync(DataChannelId, "my_channel", ordered: true, reliable: true); + + // Check consistency + Assert.AreEqual(data1.ID, data2.ID); + Assert.AreEqual(data1.Label, data2.Label); + + // Prepare for state change + var evOpen1 = new ManualResetEventSlim(initialState: false); + data1.StateChanged += () => { - await pc2.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - pc2.CreateAnswer(); + if (data1.State == DataChannel.ChannelState.Open) + { + evOpen1.Set(); + } }; - pc2.LocalSdpReadytoSend += async (string type, string sdp) => + var evOpen2 = new ManualResetEventSlim(initialState: false); + data2.StateChanged += () => { - await pc1.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - pc1.CreateAnswer(); + if (data2.State == DataChannel.ChannelState.Open) + { + evOpen2.Set(); + } }; - pc1.IceCandidateReadytoSend += (string candidate, int sdpMlineindex, string sdpMid) - => pc2.AddIceCandidate(sdpMid, sdpMlineindex, candidate); - pc2.IceCandidateReadytoSend += (string candidate, int sdpMlineindex, string sdpMid) - => pc1.AddIceCandidate(sdpMid, sdpMlineindex, candidate); // Connect + StartOfferWith(pc1_); + WaitForSdpExchangeCompleted(); + + // Wait until the data channels are ready + Assert.True(evOpen1.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(evOpen2.Wait(TimeSpan.FromSeconds(60.0))); + + // Send data { - var c1 = new ManualResetEventSlim(false); var c2 = new ManualResetEventSlim(false); - pc1.Connected += () => c1.Set(); - pc2.Connected += () => c2.Set(); - Assert.True(pc1.CreateOffer()); - Assert.True(c1.Wait(TimeSpan.FromSeconds(60.0))); + string sentText = "Some sample text"; + byte[] msg = Encoding.UTF8.GetBytes(sentText); + data2.MessageReceived += (byte[] _msg) => + { + var receivedText = Encoding.UTF8.GetString(_msg); + Assert.AreEqual(sentText, receivedText); + c2.Set(); + }; + data1.SendMessage(msg); Assert.True(c2.Wait(TimeSpan.FromSeconds(60.0))); - Assert.True(pc1.IsConnected); - Assert.True(pc1.IsConnected); } + } + + [Test] + public void SctpError() + { + // Connect + StartOfferWith(pc1_); + WaitForSdpExchangeCompleted(); // Try to add a data channel. This should fail because SCTP was not negotiated. - Assert.ThrowsAsync(async () => await pc1.AddDataChannelAsync("dummy", false, false)); - Assert.ThrowsAsync(async () => await pc1.AddDataChannelAsync(42, "dummy", false, false)); - - // Clean-up - pc1.Close(); - pc1.Dispose(); - pc2.Close(); - pc2.Dispose(); + Assert.ThrowsAsync(async () => await pc1_.AddDataChannelAsync("dummy", false, false)); + Assert.ThrowsAsync(async () => await pc1_.AddDataChannelAsync(42, "dummy", false, false)); } } } diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/LocalVideoTrackTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/LocalVideoTrackTests.cs index 2c949527d..738210486 100644 --- a/tests/Microsoft.MixedReality.WebRTC.Tests/LocalVideoTrackTests.cs +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/LocalVideoTrackTests.cs @@ -2,218 +2,19 @@ // Licensed under the MIT License. using System; -using System.Threading; +using System.Linq; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Internal; namespace Microsoft.MixedReality.WebRTC.Tests { - [TestFixture] - internal class LocalVideoTrackTests + [TestFixture(SdpSemantic.PlanB)] + [TestFixture(SdpSemantic.UnifiedPlan)] + internal class LocalVideoTrackTests : PeerConnectionTestBase { - PeerConnection pc1_ = null; - PeerConnection pc2_ = null; - ManualResetEventSlim connectedEvent1_ = null; - ManualResetEventSlim connectedEvent2_ = null; - ManualResetEventSlim renegotiationEvent1_ = null; - ManualResetEventSlim renegotiationEvent2_ = null; - ManualResetEventSlim trackAddedEvent1_ = null; - ManualResetEventSlim trackAddedEvent2_ = null; - ManualResetEventSlim trackRemovedEvent1_ = null; - ManualResetEventSlim trackRemovedEvent2_ = null; - ManualResetEventSlim iceConnectedEvent1_ = null; - ManualResetEventSlim iceConnectedEvent2_ = null; - - [SetUp] - public void SetupConnection() + public LocalVideoTrackTests(SdpSemantic sdpSemantic) : base(sdpSemantic) { - // Create the 2 peers - var config = new PeerConnectionConfiguration(); - pc1_ = new PeerConnection(); - pc2_ = new PeerConnection(); - pc1_.InitializeAsync(config).Wait(); // cannot use async/await in OneTimeSetUp - pc2_.InitializeAsync(config).Wait(); - - // Allocate callback events - connectedEvent1_ = new ManualResetEventSlim(false); - connectedEvent2_ = new ManualResetEventSlim(false); - iceConnectedEvent1_ = new ManualResetEventSlim(false); - iceConnectedEvent2_ = new ManualResetEventSlim(false); - renegotiationEvent1_ = new ManualResetEventSlim(false); - renegotiationEvent2_ = new ManualResetEventSlim(false); - trackAddedEvent1_ = new ManualResetEventSlim(false); - trackAddedEvent2_ = new ManualResetEventSlim(false); - trackRemovedEvent1_ = new ManualResetEventSlim(false); - trackRemovedEvent2_ = new ManualResetEventSlim(false); - - // Connect all signals - pc1_.Connected += OnConnected1; - pc2_.Connected += OnConnected2; - pc1_.LocalSdpReadytoSend += OnLocalSdpReady1; - pc2_.LocalSdpReadytoSend += OnLocalSdpReady2; - pc1_.IceCandidateReadytoSend += OnIceCandidateReadytoSend1; - pc2_.IceCandidateReadytoSend += OnIceCandidateReadytoSend2; - pc1_.IceStateChanged += OnIceStateChanged1; - pc2_.IceStateChanged += OnIceStateChanged2; - pc1_.RenegotiationNeeded += OnRenegotiationNeeded1; - pc2_.RenegotiationNeeded += OnRenegotiationNeeded2; - pc1_.TrackAdded += OnTrackAdded1; - pc2_.TrackAdded += OnTrackAdded2; - pc1_.TrackRemoved += OnTrackRemoved1; - pc2_.TrackRemoved += OnTrackRemoved2; - } - - [TearDown] - public void TearDownConnectin() - { - // Unregister all callbacks - pc1_.LocalSdpReadytoSend -= OnLocalSdpReady1; - pc2_.LocalSdpReadytoSend -= OnLocalSdpReady2; - pc1_.IceCandidateReadytoSend -= OnIceCandidateReadytoSend1; - pc2_.IceCandidateReadytoSend -= OnIceCandidateReadytoSend2; - pc1_.IceStateChanged -= OnIceStateChanged1; - pc2_.IceStateChanged -= OnIceStateChanged2; - pc1_.RenegotiationNeeded -= OnRenegotiationNeeded1; - pc2_.RenegotiationNeeded -= OnRenegotiationNeeded2; - pc1_.TrackAdded -= OnTrackAdded1; - pc2_.TrackAdded -= OnTrackAdded2; - pc1_.TrackRemoved -= OnTrackRemoved1; - pc2_.TrackRemoved -= OnTrackRemoved2; - - // Clean-up callback events - trackAddedEvent1_.Dispose(); - trackAddedEvent1_ = null; - trackRemovedEvent1_.Dispose(); - trackRemovedEvent1_ = null; - trackAddedEvent2_.Dispose(); - trackAddedEvent2_ = null; - trackRemovedEvent2_.Dispose(); - trackRemovedEvent2_ = null; - renegotiationEvent1_.Dispose(); - renegotiationEvent1_ = null; - renegotiationEvent2_.Dispose(); - renegotiationEvent2_ = null; - iceConnectedEvent1_.Dispose(); - iceConnectedEvent1_ = null; - iceConnectedEvent2_.Dispose(); - iceConnectedEvent2_ = null; - connectedEvent1_.Dispose(); - connectedEvent1_ = null; - connectedEvent2_.Dispose(); - connectedEvent2_ = null; - - // Destroy peers - pc1_.Close(); - pc1_.Dispose(); - pc1_ = null; - pc2_.Close(); - pc2_.Dispose(); - pc2_ = null; - } - - private void OnConnected1() - { - connectedEvent1_.Set(); - } - - private void OnConnected2() - { - connectedEvent2_.Set(); - } - - private async void OnLocalSdpReady1(string type, string sdp) - { - await pc2_.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - { - pc2_.CreateAnswer(); - } - } - - private async void OnLocalSdpReady2(string type, string sdp) - { - await pc1_.SetRemoteDescriptionAsync(type, sdp); - if (type == "offer") - { - pc1_.CreateAnswer(); - } - } - - private void OnIceCandidateReadytoSend1(string candidate, int sdpMlineindex, string sdpMid) - { - pc2_.AddIceCandidate(sdpMid, sdpMlineindex, candidate); - } - - private void OnIceCandidateReadytoSend2(string candidate, int sdpMlineindex, string sdpMid) - { - pc1_.AddIceCandidate(sdpMid, sdpMlineindex, candidate); - } - - private void OnRenegotiationNeeded1() - { - renegotiationEvent1_.Set(); - if (pc1_.IsConnected) - { - pc1_.CreateOffer(); - } - } - - private void OnRenegotiationNeeded2() - { - renegotiationEvent2_.Set(); - if (pc2_.IsConnected) - { - pc2_.CreateOffer(); - } - } - - private void OnTrackAdded1(PeerConnection.TrackKind trackKind) - { - Assert.True(trackKind == PeerConnection.TrackKind.Video); - trackAddedEvent1_.Set(); - } - - private void OnTrackAdded2(PeerConnection.TrackKind trackKind) - { - Assert.True(trackKind == PeerConnection.TrackKind.Video); - trackAddedEvent2_.Set(); - } - - private void OnTrackRemoved1(PeerConnection.TrackKind trackKind) - { - Assert.True(trackKind == PeerConnection.TrackKind.Video); - trackRemovedEvent1_.Set(); - } - - private void OnTrackRemoved2(PeerConnection.TrackKind trackKind) - { - Assert.True(trackKind == PeerConnection.TrackKind.Video); - trackRemovedEvent2_.Set(); - } - - private void OnIceStateChanged1(IceConnectionState newState) - { - if ((newState == IceConnectionState.Connected) || (newState == IceConnectionState.Completed)) - { - iceConnectedEvent1_.Set(); - } - } - - private void OnIceStateChanged2(IceConnectionState newState) - { - if ((newState == IceConnectionState.Connected) || (newState == IceConnectionState.Completed)) - { - iceConnectedEvent2_.Set(); - } - } - - private void WaitForSdpExchangeCompleted() - { - Assert.True(connectedEvent1_.Wait(TimeSpan.FromSeconds(60.0))); - Assert.True(connectedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); - connectedEvent1_.Reset(); - connectedEvent2_.Reset(); } private unsafe void CustomI420AFrameCallback(in FrameRequest request) @@ -301,137 +102,269 @@ private unsafe void CustomArgb32FrameCallback(in FrameRequest request) #if !MRSW_EXCLUDE_DEVICE_TESTS [Test] - public async Task AddLocalVideoTrackAsync_Null() + public async Task BeforeConnect() { - LocalVideoTrack track1 = await pc1_.AddLocalVideoTrackAsync(); - Assert.NotNull(track1); - Assert.AreEqual(pc1_, track1.PeerConnection); - pc1_.RemoveLocalVideoTrack(track1); - track1.Dispose(); - } + // Create video transceiver on #1 + var transceiver_settings = new TransceiverInitSettings + { + Name = "transceiver1", + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind.Video, transceiver_settings); + Assert.NotNull(transceiver1); - [Test] - public async Task AddLocalVideoTrackAsync_Default() - { + // Wait for local SDP re-negotiation event on #1. + // This will not create an offer, since we're not connected yet. + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + + // Create local video track var settings = new LocalVideoTrackSettings(); - LocalVideoTrack track1 = await pc1_.AddLocalVideoTrackAsync(settings); + LocalVideoTrack track1 = await LocalVideoTrack.CreateFromDeviceAsync(settings); Assert.NotNull(track1); - Assert.AreEqual(pc1_, track1.PeerConnection); - pc1_.RemoveLocalVideoTrack(track1); - track1.Dispose(); - } - [Test] - public async Task BeforeConnect() - { // Add local video track channel to #1 - var settings = new LocalVideoTrackSettings(); - LocalVideoTrack track1 = await pc1_.AddLocalVideoTrackAsync(settings); - Assert.NotNull(track1); + renegotiationEvent1_.Reset(); + transceiver1.LocalVideoTrack = track1; + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed Assert.AreEqual(pc1_, track1.PeerConnection); - - // Wait for local SDP re-negotiation on #1. - // This will not create an offer, since we're not connected yet. - Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.AreEqual(track1, transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); + Assert.IsTrue(pc1_.LocalVideoTracks.Contains(track1)); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); // Connect - Assert.True(pc1_.CreateOffer()); - WaitForSdpExchangeCompleted(); + StartOfferWith(pc1_); + WaitForTransportsWritable(); Assert.True(pc1_.IsConnected); Assert.True(pc2_.IsConnected); + // Now remote peer #2 has a 1 remote track + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(1, pc2_.RemoteVideoTracks.Count()); + + // Wait until the SDP exchange is completed + WaitForSdpExchangeCompleted(); + // Remove the track from #1 renegotiationEvent1_.Reset(); - pc1_.RemoveLocalVideoTrack(track1); + transceiver1.LocalVideoTrack = null; + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); // never removed + Assert.IsNull(transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); track1.Dispose(); - // Wait for local SDP re-negotiation on #1 - Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + // Remote peer #2 still has a track, because the transceiver is still receiving, + // even if there is no track on the sending side (so effectively it receives only + // black frames). + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(1, pc2_.RemoteVideoTracks.Count()); - // Confirm remote track was removed from #2 - Assert.True(trackRemovedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + // Change the transceiver direction to stop receiving. This requires a renegotiation + // to take effect, so nothing changes for now. + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + transceiver1.DesiredDirection = Transceiver.Direction.Inactive; - // Wait until SDP renegotiation finished + // Wait for renegotiate to complete + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); WaitForSdpExchangeCompleted(); + Assert.True(remoteDescAppliedEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(remoteDescAppliedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + + // Now the remote track got removed from #2 + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); } [Test] public async Task AfterConnect() { // Connect - Assert.True(pc1_.CreateOffer()); - WaitForSdpExchangeCompleted(); + StartOfferWith(pc1_); + WaitForTransportsWritable(); Assert.True(pc1_.IsConnected); Assert.True(pc2_.IsConnected); - // Add local video track channel to #1 - var settings = new LocalVideoTrackSettings(); - LocalVideoTrack track1 = await pc1_.AddLocalVideoTrackAsync(settings); - Assert.NotNull(track1); - Assert.AreEqual(pc1_, track1.PeerConnection); - - // Wait for local SDP re-negotiation on #1 - Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + // Wait for all transceivers to be updated on both peers + WaitForSdpExchangeCompleted(); + Assert.True(remoteDescAppliedEvent1_.Wait(TimeSpan.FromSeconds(20.0))); + Assert.True(remoteDescAppliedEvent2_.Wait(TimeSpan.FromSeconds(20.0))); + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + + // No track yet + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); + + // Create video transceiver on #1 -- this generates a renegotiation + renegotiationEvent1_.Reset(); + var transceiver_settings = new TransceiverInitSettings + { + Name = "transceiver1", + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind.Video, transceiver_settings); + Assert.NotNull(transceiver1); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); - // Confirm remote track was added on #2 - Assert.True(trackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + // Confirm (inactive) remote track was added on #2 due to transceiver being added + Assert.True(videoTrackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); // Wait until SDP renegotiation finished + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); WaitForSdpExchangeCompleted(); + // Now remote peer #2 has a 1 remote track (which is inactive). + // Note that tracks are updated before transceivers here. This might be unintuitive, so we might + // want to revisit this later. + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(1, pc2_.RemoteVideoTracks.Count()); + + // Transceiver has been updated to Send+Receive (default desired direction when added), but + // since peer #2 doesn't intend to send, the actually negotiated direction on #1 is Send only. + Assert.AreEqual(Transceiver.Direction.SendReceive, transceiver1.DesiredDirection); + Assert.AreEqual(Transceiver.Direction.SendOnly, transceiver1.NegotiatedDirection); + + // Create local track + renegotiationEvent1_.Reset(); + var settings = new LocalVideoTrackSettings(); + LocalVideoTrack track1 = await LocalVideoTrack.CreateFromDeviceAsync(settings); + Assert.NotNull(track1); + Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed + + // Add local video track to #1 + renegotiationEvent1_.Reset(); + transceiver1.LocalVideoTrack = track1; + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed + Assert.AreEqual(pc1_, track1.PeerConnection); + Assert.AreEqual(track1, transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); + Assert.IsTrue(pc1_.LocalVideoTracks.Contains(track1)); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + + // SetLocalTrack() does not change the transceiver directions + Assert.AreEqual(Transceiver.Direction.SendReceive, transceiver1.DesiredDirection); + Assert.AreEqual(Transceiver.Direction.SendOnly, transceiver1.NegotiatedDirection); + // Remove the track from #1 renegotiationEvent1_.Reset(); - pc1_.RemoveLocalVideoTrack(track1); + transceiver1.LocalVideoTrack = null; + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); // never removed + Assert.IsNull(transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); track1.Dispose(); - // Wait for local SDP re-negotiation on #1 + // SetLocalTrack() does not change the transceiver directions, even when the local + // sending track is disposed of. + Assert.AreEqual(Transceiver.Direction.SendReceive, transceiver1.DesiredDirection); + Assert.AreEqual(Transceiver.Direction.SendOnly, transceiver1.NegotiatedDirection); + + // Remote peer #2 still has a track, because the transceiver is still receiving, + // even if there is no track on the sending side (so effectively it receives only + // black frames). + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(1, pc2_.RemoteVideoTracks.Count()); + + // Change the transceiver direction to stop receiving. This requires a renegotiation + // to take effect, so nothing changes for now. + // Note: In Plan B, a renegotiation needed event is manually forced for parity with + // Unified Plan. However setting the transceiver to inactive removes the remote peer's + // remote track, which causes another renegotiation needed event. So we suspend the + // automatic offer to trigger it manually. + suspendOffer1_ = true; + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + transceiver1.DesiredDirection = Transceiver.Direction.Inactive; Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); - // Confirm remote track was removed from #2 -- not fired in Unified Plan - //Assert.True(trackRemovedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); - - // Wait until SDP renegotiation finished + // Wait for renegotiate to complete + StartOfferWith(pc1_); WaitForSdpExchangeCompleted(); + + // Now the remote track got removed from #2 + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); } #endif // !MRSW_EXCLUDE_DEVICE_TESTS - [Test] public void SimpleExternalI420A() { // Connect - Assert.True(pc1_.CreateOffer()); - WaitForSdpExchangeCompleted(); + StartOfferWith(pc1_); + WaitForTransportsWritable(); Assert.True(pc1_.IsConnected); Assert.True(pc2_.IsConnected); + WaitForSdpExchangeCompleted(); - // Create external ARGB32 source + // No track yet + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); + + // Create external I420A source var source1 = ExternalVideoTrackSource.CreateFromI420ACallback(CustomI420AFrameCallback); Assert.NotNull(source1); - Assert.AreEqual(null, source1.PeerConnection); // before any track was added, this is null + Assert.AreEqual(0, source1.Tracks.Count()); - // Add external ARGB32 track - var track1 = pc1_.AddCustomLocalVideoTrack("custom_i420a", source1); - Assert.NotNull(track1); - Assert.AreEqual(pc1_, track1.PeerConnection); - Assert.AreEqual(pc1_, source1.PeerConnection); // after a track was added, this is the PC of the track + // Add video transceiver #1 + renegotiationEvent1_.Reset(); + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + Assert.IsFalse(videoTrackAddedEvent2_.IsSet); + var transceiver_settings = new TransceiverInitSettings + { + Name = "transceiver1", + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind.Video, transceiver_settings); + Assert.NotNull(transceiver1); + Assert.IsNull(transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); + Assert.AreEqual(pc1_, transceiver1.PeerConnection); - // Wait for local SDP re-negotiation on #1 + // Wait for renegotiation Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(videoTrackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(remoteDescAppliedEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(remoteDescAppliedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + WaitForSdpExchangeCompleted(); - // Confirm remote track was added on #2 - Assert.True(trackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + // Create external I420A track + var track1 = LocalVideoTrack.CreateFromExternalSource("custom_i420a", source1); + Assert.NotNull(track1); + Assert.AreEqual(source1, track1.Source); + Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.IsFalse(pc1_.LocalVideoTracks.Contains(track1)); - // Wait until SDP renegotiation finished - WaitForSdpExchangeCompleted(); + // Set track on transceiver + renegotiationEvent1_.Reset(); + transceiver1.LocalVideoTrack = track1; + Assert.AreEqual(pc1_, track1.PeerConnection); + Assert.IsTrue(pc1_.LocalVideoTracks.Contains(track1)); + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed // Remove the track from #1 renegotiationEvent1_.Reset(); - pc1_.RemoveLocalVideoTrack(track1); + transceiver1.LocalVideoTrack = null; Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.IsFalse(pc1_.LocalVideoTracks.Contains(track1)); // Dispose of the track and its source track1.Dispose(); @@ -439,49 +372,75 @@ public void SimpleExternalI420A() source1.Dispose(); source1 = null; - // Wait for local SDP re-negotiation on #1 - Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); - - // Confirm remote track was removed from #2 - Assert.True(trackRemovedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); - - // Wait until SDP renegotiation finished - WaitForSdpExchangeCompleted(); + // On peer #1 the track was replaced on the transceiver, but the transceiver stays + // on the peer connection, so no renegotiation is needed. + Assert.IsFalse(renegotiationEvent1_.IsSet); } [Test] public void SimpleExternalArgb32() { // Connect - Assert.True(pc1_.CreateOffer()); - WaitForSdpExchangeCompleted(); + StartOfferWith(pc1_); + WaitForTransportsWritable(); Assert.True(pc1_.IsConnected); Assert.True(pc2_.IsConnected); + WaitForSdpExchangeCompleted(); + + // No track yet + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); // Create external ARGB32 source var source1 = ExternalVideoTrackSource.CreateFromArgb32Callback(CustomArgb32FrameCallback); Assert.NotNull(source1); - Assert.AreEqual(null, source1.PeerConnection); // before any track was added, this is null + Assert.AreEqual(0, source1.Tracks.Count()); - // Add external ARGB32 track - var track1 = pc1_.AddCustomLocalVideoTrack("custom_argb32", source1); - Assert.NotNull(track1); - Assert.AreEqual(pc1_, track1.PeerConnection); - Assert.AreEqual(pc1_, source1.PeerConnection); // after a track was added, this is the PC of the track + // Add video transceiver #1 + renegotiationEvent1_.Reset(); + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + Assert.IsFalse(videoTrackAddedEvent2_.IsSet); + var transceiver_settings = new TransceiverInitSettings + { + Name = "transceiver1", + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind.Video, transceiver_settings); + Assert.NotNull(transceiver1); + Assert.IsNull(transceiver1.LocalTrack); + Assert.IsNull(transceiver1.RemoteTrack); + Assert.AreEqual(pc1_, transceiver1.PeerConnection); - // Wait for local SDP re-negotiation on #1 + // Wait for renegotiation Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(videoTrackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(remoteDescAppliedEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.True(remoteDescAppliedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + WaitForSdpExchangeCompleted(); - // Confirm remote track was added on #2 - Assert.True(trackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + // Create external ARGB32 track + var track1 = LocalVideoTrack.CreateFromExternalSource("custom_argb32", source1); + Assert.NotNull(track1); + Assert.AreEqual(source1, track1.Source); + Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.IsFalse(pc1_.LocalVideoTracks.Contains(track1)); - // Wait until SDP renegotiation finished - WaitForSdpExchangeCompleted(); + // Set track on transceiver + renegotiationEvent1_.Reset(); + transceiver1.LocalVideoTrack = track1; + Assert.AreEqual(pc1_, track1.PeerConnection); + Assert.IsTrue(pc1_.LocalVideoTracks.Contains(track1)); + Assert.IsFalse(renegotiationEvent1_.IsSet); // renegotiation not needed // Remove the track from #1 renegotiationEvent1_.Reset(); - pc1_.RemoveLocalVideoTrack(track1); + transceiver1.LocalVideoTrack = null; Assert.IsNull(track1.PeerConnection); + Assert.IsNull(track1.Transceiver); + Assert.IsFalse(pc1_.LocalVideoTracks.Contains(track1)); // Dispose of the track and its source track1.Dispose(); @@ -489,14 +448,120 @@ public void SimpleExternalArgb32() source1.Dispose(); source1 = null; + // On peer #1 the track was replaced on the transceiver, but the transceiver stays + // on the peer connection, so no renegotiation is needed. + Assert.IsFalse(renegotiationEvent1_.IsSet); + } + + [Test] + public void MultiExternalI420A() + { + // Batch changes in this test, and manually (re)negotiate + suspendOffer1_ = true; + + // Connect + StartOfferWith(pc1_); + WaitForTransportsWritable(); + Assert.True(pc1_.IsConnected); + Assert.True(pc2_.IsConnected); + WaitForSdpExchangeCompleted(); + + // No track yet + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); + + // Create external I420A source + var source1 = ExternalVideoTrackSource.CreateFromI420ACallback(CustomI420AFrameCallback); + Assert.NotNull(source1); + Assert.AreEqual(0, source1.Tracks.Count()); + + // Add external I420A tracks + const int kNumTracks = 5; + var transceivers = new Transceiver[kNumTracks]; + var tracks = new LocalVideoTrack[kNumTracks]; + for (int i = 0; i < kNumTracks; ++i) + { + var transceiver_settings = new TransceiverInitSettings + { + Name = $"transceiver1_{i}", + }; + transceivers[i] = pc1_.AddTransceiver(MediaKind.Video, transceiver_settings); + Assert.NotNull(transceivers[i]); + + tracks[i] = LocalVideoTrack.CreateFromExternalSource($"track_i420a_{i}", source1); + Assert.NotNull(tracks[i]); + Assert.IsTrue(source1.Tracks.Contains(tracks[i])); + + transceivers[i].LocalVideoTrack = tracks[i]; + Assert.AreEqual(pc1_, tracks[i].PeerConnection); + Assert.IsTrue(pc1_.LocalVideoTracks.Contains(tracks[i])); + } + Assert.AreEqual(kNumTracks, source1.Tracks.Count()); + Assert.AreEqual(kNumTracks, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + // Wait for local SDP re-negotiation on #1 Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); - // Confirm remote track was removed from #2 - Assert.True(trackRemovedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + // Renegotiate + StartOfferWith(pc1_); + + // Confirm remote track was added on #2 + Assert.True(videoTrackAddedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); // Wait until SDP renegotiation finished WaitForSdpExchangeCompleted(); + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(kNumTracks, pc2_.RemoteVideoTracks.Count()); + + // Remove the track from #1 + renegotiationEvent1_.Reset(); + for (int i = 0; i < kNumTracks; ++i) + { + transceivers[i].LocalVideoTrack = null; + Assert.IsNull(tracks[i].PeerConnection); + Assert.IsFalse(pc1_.LocalVideoTracks.Contains(tracks[i])); + Assert.IsTrue(source1.Tracks.Contains(tracks[i])); // does not change yet + tracks[i].Dispose(); + tracks[i] = null; + Assert.IsFalse(source1.Tracks.Contains(tracks[i])); + } + Assert.AreEqual(0, pc1_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc1_.RemoteVideoTracks.Count()); + Assert.AreEqual(0, source1.Tracks.Count()); + + // Dispose of source + source1.Dispose(); + source1 = null; + + // Changes on transceiver's local track do not require renegotiation + Assert.False(renegotiationEvent1_.IsSet); + + // Change the transceivers direction to stop sending + renegotiationEvent1_.Reset(); + videoTrackRemovedEvent2_.Reset(); + remoteDescAppliedEvent1_.Reset(); + remoteDescAppliedEvent2_.Reset(); + for (int i = 0; i < kNumTracks; ++i) + { + transceivers[i].DesiredDirection = Transceiver.Direction.Inactive; + } + + // Renegotiate manually the batch of changes + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + StartOfferWith(pc1_); + + // Wait everything to be ready + WaitForSdpExchangeCompleted(); + + // Confirm remote tracks were removed from #2 as part of removing all transceivers + Assert.True(videoTrackRemovedEvent2_.IsSet); + + // Remote peer #2 doesn't have any track anymore + Assert.AreEqual(0, pc2_.LocalVideoTracks.Count()); + Assert.AreEqual(0, pc2_.RemoteVideoTracks.Count()); } } } diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTestBase.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTestBase.cs new file mode 100644 index 000000000..e6fd65621 --- /dev/null +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTestBase.cs @@ -0,0 +1,396 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace Microsoft.MixedReality.WebRTC.Tests +{ + /// + /// Base class for most C# tests. + /// + /// This helper class provides several features: + /// - Allows testing the various SDP semantics. + /// - Ensures there is no live object before AND after the test. + /// - Sets up and tears down the various event handlers for the most + /// common events, to avoid duplication in all tests. + /// + internal class PeerConnectionTestBase + { + /// + /// SDP semantic for the current test case. + /// + protected readonly SdpSemantic sdpSemantic_; + + protected PeerConnection pc1_ = null; + protected PeerConnection pc2_ = null; + protected bool exchangePending_ = false; + protected ManualResetEventSlim exchangeCompleted_ = null; + protected ManualResetEventSlim connectedEvent1_ = null; + protected ManualResetEventSlim connectedEvent2_ = null; + protected ManualResetEventSlim remoteDescAppliedEvent1_ = null; + protected ManualResetEventSlim remoteDescAppliedEvent2_ = null; + protected ManualResetEventSlim renegotiationEvent1_ = null; + protected ManualResetEventSlim renegotiationEvent2_ = null; + protected ManualResetEventSlim dataChannelAddedEvent1_ = null; + protected ManualResetEventSlim dataChannelAddedEvent2_ = null; + protected ManualResetEventSlim dataChannelRemovedEvent1_ = null; + protected ManualResetEventSlim dataChannelRemovedEvent2_ = null; + protected ManualResetEventSlim audioTrackAddedEvent1_ = null; + protected ManualResetEventSlim audioTrackAddedEvent2_ = null; + protected ManualResetEventSlim audioTrackRemovedEvent1_ = null; + protected ManualResetEventSlim audioTrackRemovedEvent2_ = null; + protected ManualResetEventSlim videoTrackAddedEvent1_ = null; + protected ManualResetEventSlim videoTrackAddedEvent2_ = null; + protected ManualResetEventSlim videoTrackRemovedEvent1_ = null; + protected ManualResetEventSlim videoTrackRemovedEvent2_ = null; + protected ManualResetEventSlim iceConnectedEvent1_ = null; + protected ManualResetEventSlim iceConnectedEvent2_ = null; + protected bool suspendOffer1_ = false; + protected bool suspendOffer2_ = false; + + public PeerConnectionTestBase(SdpSemantic sdpSemantic) + { + sdpSemantic_ = sdpSemantic; + } + + [SetUp] + public void SetupConnection() + { + Assert.AreEqual(0, Library.ReportLiveObjects()); + + // Create the 2 peers + var config = new PeerConnectionConfiguration(); + config.SdpSemantic = sdpSemantic_; + pc1_ = new PeerConnection(); + pc2_ = new PeerConnection(); + pc1_.InitializeAsync(config).Wait(); // cannot use async/await in OneTimeSetUp + pc2_.InitializeAsync(config).Wait(); + + exchangePending_ = false; + exchangeCompleted_ = new ManualResetEventSlim(false); + + // Allocate callback events + connectedEvent1_ = new ManualResetEventSlim(false); + connectedEvent2_ = new ManualResetEventSlim(false); + remoteDescAppliedEvent1_ = new ManualResetEventSlim(false); + remoteDescAppliedEvent2_ = new ManualResetEventSlim(false); + iceConnectedEvent1_ = new ManualResetEventSlim(false); + iceConnectedEvent2_ = new ManualResetEventSlim(false); + renegotiationEvent1_ = new ManualResetEventSlim(false); + renegotiationEvent2_ = new ManualResetEventSlim(false); + dataChannelAddedEvent1_ = new ManualResetEventSlim(false); + dataChannelAddedEvent2_ = new ManualResetEventSlim(false); + dataChannelRemovedEvent1_ = new ManualResetEventSlim(false); + dataChannelRemovedEvent2_ = new ManualResetEventSlim(false); + audioTrackAddedEvent1_ = new ManualResetEventSlim(false); + audioTrackAddedEvent2_ = new ManualResetEventSlim(false); + audioTrackRemovedEvent1_ = new ManualResetEventSlim(false); + audioTrackRemovedEvent2_ = new ManualResetEventSlim(false); + videoTrackAddedEvent1_ = new ManualResetEventSlim(false); + videoTrackAddedEvent2_ = new ManualResetEventSlim(false); + videoTrackRemovedEvent1_ = new ManualResetEventSlim(false); + videoTrackRemovedEvent2_ = new ManualResetEventSlim(false); + + // Connect all signals + pc1_.Connected += OnConnected1; + pc2_.Connected += OnConnected2; + pc1_.LocalSdpReadytoSend += OnLocalSdpReady1; + pc2_.LocalSdpReadytoSend += OnLocalSdpReady2; + pc1_.IceCandidateReadytoSend += OnIceCandidateReadytoSend1; + pc2_.IceCandidateReadytoSend += OnIceCandidateReadytoSend2; + pc1_.IceStateChanged += OnIceStateChanged1; + pc2_.IceStateChanged += OnIceStateChanged2; + pc1_.RenegotiationNeeded += OnRenegotiationNeeded1; + pc2_.RenegotiationNeeded += OnRenegotiationNeeded2; + pc1_.DataChannelAdded += OnDataChannelAdded1; + pc2_.DataChannelAdded += OnDataChannelAdded2; + pc1_.DataChannelRemoved += OnDataChannelRemoved1; + pc2_.DataChannelRemoved += OnDataChannelRemoved2; + pc1_.AudioTrackAdded += OnAudioTrackAdded1; + pc2_.AudioTrackAdded += OnAudioTrackAdded2; + pc1_.AudioTrackRemoved += OnAudioTrackRemoved1; + pc2_.AudioTrackRemoved += OnAudioTrackRemoved2; + pc1_.VideoTrackAdded += OnVideoTrackAdded1; + pc2_.VideoTrackAdded += OnVideoTrackAdded2; + pc1_.VideoTrackRemoved += OnVideoTrackRemoved1; + pc2_.VideoTrackRemoved += OnVideoTrackRemoved2; + + // Enable automatic renegotiation + suspendOffer1_ = false; + suspendOffer2_ = false; + } + + [TearDown] + public void TearDownConnectin() + { + // Unregister all callbacks + pc1_.LocalSdpReadytoSend -= OnLocalSdpReady1; + pc2_.LocalSdpReadytoSend -= OnLocalSdpReady2; + pc1_.IceCandidateReadytoSend -= OnIceCandidateReadytoSend1; + pc2_.IceCandidateReadytoSend -= OnIceCandidateReadytoSend2; + pc1_.IceStateChanged -= OnIceStateChanged1; + pc2_.IceStateChanged -= OnIceStateChanged2; + pc1_.RenegotiationNeeded -= OnRenegotiationNeeded1; + pc2_.RenegotiationNeeded -= OnRenegotiationNeeded2; + pc1_.DataChannelAdded -= OnDataChannelAdded1; + pc2_.DataChannelAdded -= OnDataChannelAdded2; + pc1_.DataChannelRemoved -= OnDataChannelRemoved1; + pc2_.DataChannelRemoved -= OnDataChannelRemoved2; + pc1_.AudioTrackAdded -= OnAudioTrackAdded1; + pc2_.AudioTrackAdded -= OnAudioTrackAdded2; + pc1_.AudioTrackRemoved -= OnAudioTrackRemoved1; + pc2_.AudioTrackRemoved -= OnAudioTrackRemoved2; + pc1_.VideoTrackAdded -= OnVideoTrackAdded1; + pc2_.VideoTrackAdded -= OnVideoTrackAdded2; + pc1_.VideoTrackRemoved -= OnVideoTrackRemoved1; + pc2_.VideoTrackRemoved -= OnVideoTrackRemoved2; + + Assert.IsFalse(exchangePending_); + exchangeCompleted_.Dispose(); + exchangeCompleted_ = null; + + // Clean-up callback events + dataChannelAddedEvent1_.Dispose(); + dataChannelAddedEvent1_ = null; + dataChannelRemovedEvent1_.Dispose(); + dataChannelRemovedEvent1_ = null; + audioTrackAddedEvent1_.Dispose(); + audioTrackAddedEvent1_ = null; + audioTrackRemovedEvent1_.Dispose(); + audioTrackRemovedEvent1_ = null; + audioTrackAddedEvent2_.Dispose(); + audioTrackAddedEvent2_ = null; + audioTrackRemovedEvent2_.Dispose(); + audioTrackRemovedEvent2_ = null; + videoTrackAddedEvent1_.Dispose(); + videoTrackAddedEvent1_ = null; + videoTrackRemovedEvent1_.Dispose(); + videoTrackRemovedEvent1_ = null; + videoTrackAddedEvent2_.Dispose(); + videoTrackAddedEvent2_ = null; + videoTrackRemovedEvent2_.Dispose(); + videoTrackRemovedEvent2_ = null; + renegotiationEvent1_.Dispose(); + renegotiationEvent1_ = null; + renegotiationEvent2_.Dispose(); + renegotiationEvent2_ = null; + iceConnectedEvent1_.Dispose(); + iceConnectedEvent1_ = null; + iceConnectedEvent2_.Dispose(); + iceConnectedEvent2_ = null; + remoteDescAppliedEvent1_.Dispose(); + remoteDescAppliedEvent1_ = null; + remoteDescAppliedEvent2_.Dispose(); + remoteDescAppliedEvent2_ = null; + connectedEvent1_.Dispose(); + connectedEvent1_ = null; + connectedEvent2_.Dispose(); + connectedEvent2_ = null; + + // Destroy peers + pc1_.Close(); + pc1_.Dispose(); + pc1_ = null; + pc2_.Close(); + pc2_.Dispose(); + pc2_ = null; + + Assert.AreEqual(0, Library.ReportLiveObjects()); + } + + /// + /// Start an SDP exchange by sending an offer from the given peer. + /// + /// The peer to call on. + protected void StartOfferWith(PeerConnection offeringPeer) + { + Assert.IsFalse(exchangePending_); + exchangePending_ = true; + exchangeCompleted_.Reset(); + connectedEvent1_.Reset(); + connectedEvent2_.Reset(); + Assert.IsTrue(offeringPeer.CreateOffer()); + } + + /// + /// Wait until transports are writable. This is not the end of the SDP + /// exchange, but transceivers are starting to send/receive. The offer + /// was accepted, but the offering peer has yet to receive and apply an + /// SDP answer though. + /// + protected void WaitForTransportsWritable() + { + Assert.IsTrue(exchangePending_); + Assert.IsTrue(connectedEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.IsTrue(connectedEvent2_.Wait(TimeSpan.FromSeconds(60.0))); + connectedEvent1_.Reset(); + connectedEvent2_.Reset(); + } + + /// + /// Wait until the SDP exchange finally completed and the answer has been + /// applied back on the offering peer. + /// + protected void WaitForSdpExchangeCompleted() + { + Assert.IsTrue(exchangeCompleted_.Wait(TimeSpan.FromSeconds(60.0))); + Assert.IsFalse(exchangePending_); + exchangeCompleted_.Reset(); + } + + protected Task DoNegotiationStartFrom(PeerConnection offeringPeer) + { + StartOfferWith(offeringPeer); + return Task.Run(() => { WaitForSdpExchangeCompleted(); }); + } + + private void OnConnected1() + { + connectedEvent1_.Set(); + } + + private void OnConnected2() + { + connectedEvent2_.Set(); + } + + private async void OnLocalSdpReady1(string type, string sdp) + { + Assert.IsTrue(exchangePending_); + await pc2_.SetRemoteDescriptionAsync(type, sdp); + remoteDescAppliedEvent2_.Set(); + if (type == "offer") + { + pc2_.CreateAnswer(); + } + else + { + exchangePending_ = false; + exchangeCompleted_.Set(); + } + } + + private async void OnLocalSdpReady2(string type, string sdp) + { + Assert.IsTrue(exchangePending_); + await pc1_.SetRemoteDescriptionAsync(type, sdp); + remoteDescAppliedEvent1_.Set(); + if (type == "offer") + { + pc1_.CreateAnswer(); + } + else + { + exchangePending_ = false; + exchangeCompleted_.Set(); + } + } + + private void OnIceCandidateReadytoSend1(string candidate, int sdpMlineindex, string sdpMid) + { + pc2_.AddIceCandidate(sdpMid, sdpMlineindex, candidate); + } + + private void OnIceCandidateReadytoSend2(string candidate, int sdpMlineindex, string sdpMid) + { + pc1_.AddIceCandidate(sdpMid, sdpMlineindex, candidate); + } + + private void OnRenegotiationNeeded1() + { + renegotiationEvent1_.Set(); + if (pc1_.IsConnected && !suspendOffer1_) + { + StartOfferWith(pc1_); + } + } + + private void OnRenegotiationNeeded2() + { + renegotiationEvent2_.Set(); + if (pc2_.IsConnected && !suspendOffer2_) + { + StartOfferWith(pc2_); + } + } + + private void OnAudioTrackAdded1(RemoteAudioTrack track) + { + audioTrackAddedEvent1_.Set(); + } + + private void OnAudioTrackAdded2(RemoteAudioTrack track) + { + audioTrackAddedEvent2_.Set(); + } + + private void OnAudioTrackRemoved1(Transceiver transceiver, RemoteAudioTrack track) + { + audioTrackRemovedEvent1_.Set(); + } + + private void OnAudioTrackRemoved2(Transceiver transceiver, RemoteAudioTrack track) + { + audioTrackRemovedEvent2_.Set(); + } + + private void OnVideoTrackAdded1(RemoteVideoTrack track) + { + videoTrackAddedEvent1_.Set(); + } + + private void OnVideoTrackAdded2(RemoteVideoTrack track) + { + videoTrackAddedEvent2_.Set(); + } + + private void OnVideoTrackRemoved1(Transceiver transceiver, RemoteVideoTrack track) + { + videoTrackRemovedEvent1_.Set(); + } + + private void OnVideoTrackRemoved2(Transceiver transceiver, RemoteVideoTrack track) + { + videoTrackRemovedEvent2_.Set(); + } + + private void OnIceStateChanged1(IceConnectionState newState) + { + if ((newState == IceConnectionState.Connected) || (newState == IceConnectionState.Completed)) + { + iceConnectedEvent1_.Set(); + } + } + + private void OnIceStateChanged2(IceConnectionState newState) + { + if ((newState == IceConnectionState.Connected) || (newState == IceConnectionState.Completed)) + { + iceConnectedEvent2_.Set(); + } + } + + private void OnDataChannelAdded1(DataChannel channel) + { + dataChannelAddedEvent1_.Set(); + } + + private void OnDataChannelAdded2(DataChannel channel) + { + dataChannelAddedEvent2_.Set(); + } + + private void OnDataChannelRemoved1(DataChannel channel) + { + dataChannelRemovedEvent1_.Set(); + } + + private void OnDataChannelRemoved2(DataChannel channel) + { + dataChannelRemovedEvent2_.Set(); + } + } +} diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTests.cs index 6bf34d12b..7321ef354 100644 --- a/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTests.cs +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/PeerConnectionTests.cs @@ -17,6 +17,7 @@ public async Task LocalNoICE() var pc1 = new PeerConnection(); var pc2 = new PeerConnection(); + var evExchangeCompleted = new ManualResetEventSlim(initialState: false); pc1.LocalSdpReadytoSend += async (string type, string sdp) => { await pc2.SetRemoteDescriptionAsync(type, sdp); @@ -24,6 +25,10 @@ public async Task LocalNoICE() { pc2.CreateAnswer(); } + else + { + evExchangeCompleted.Set(); + } }; pc2.LocalSdpReadytoSend += async (string type, string sdp) => { @@ -32,6 +37,10 @@ public async Task LocalNoICE() { pc1.CreateAnswer(); } + else + { + evExchangeCompleted.Set(); + } }; var pcConfig = new PeerConnectionConfiguration(); @@ -40,8 +49,10 @@ public async Task LocalNoICE() var ev1 = new ManualResetEventSlim(initialState: false); pc1.Connected += () => ev1.Set(); + evExchangeCompleted.Reset(); pc1.CreateOffer(); ev1.Wait(millisecondsTimeout: 5000); + evExchangeCompleted.Wait(millisecondsTimeout: 5000); pc1.Close(); pc2.Close(); @@ -49,6 +60,7 @@ public async Task LocalNoICE() protected async Task MakeICECall(PeerConnection pc1, PeerConnection pc2) { + var evExchangeCompleted = new ManualResetEventSlim(initialState: false); pc1.LocalSdpReadytoSend += async (string type, string sdp) => { await pc2.SetRemoteDescriptionAsync(type, sdp); @@ -56,6 +68,10 @@ protected async Task MakeICECall(PeerConnection pc1, PeerConnection pc2) { pc2.CreateAnswer(); } + else + { + evExchangeCompleted.Set(); + } }; pc2.LocalSdpReadytoSend += async (string type, string sdp) => { @@ -64,6 +80,10 @@ protected async Task MakeICECall(PeerConnection pc1, PeerConnection pc2) { pc1.CreateAnswer(); } + else + { + evExchangeCompleted.Set(); + } }; pc1.IceCandidateReadytoSend += (string candidate, int sdpMlineindex, string sdpMid) => { @@ -82,9 +102,11 @@ protected async Task MakeICECall(PeerConnection pc1, PeerConnection pc2) var ev2 = new ManualResetEventSlim(initialState: false); pc1.Connected += () => ev1.Set(); pc2.Connected += () => ev2.Set(); + evExchangeCompleted.Reset(); pc1.CreateOffer(); ev1.Wait(millisecondsTimeout: 5000); ev2.Wait(millisecondsTimeout: 5000); + evExchangeCompleted.Wait(millisecondsTimeout: 5000); } [Test] diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/TransceiverTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/TransceiverTests.cs new file mode 100644 index 000000000..0595219d5 --- /dev/null +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/TransceiverTests.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using NUnit.Framework.Internal; + +namespace Microsoft.MixedReality.WebRTC.Tests +{ + [TestFixture(SdpSemantic.PlanB, MediaKind.Audio)] + [TestFixture(SdpSemantic.UnifiedPlan, MediaKind.Audio)] + [TestFixture(SdpSemantic.PlanB, MediaKind.Video)] + [TestFixture(SdpSemantic.UnifiedPlan, MediaKind.Video)] + internal class TransceiverTests : PeerConnectionTestBase + { + private readonly MediaKind MediaKind; + + public TransceiverTests(SdpSemantic sdpSemantic, MediaKind mediaKind) : base(sdpSemantic) + { + MediaKind = mediaKind; + } + + [Test] + public async Task SetDirection() + { + // This test use manual offers + suspendOffer1_ = true; + + // Create video transceiver on #1. This triggers a renegotiation needed event. + var transceiver_settings = new TransceiverInitSettings + { + Name = "transceiver1", + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind, transceiver_settings); + Assert.NotNull(transceiver1); + Assert.AreEqual(transceiver1.DesiredDirection, Transceiver.Direction.SendReceive); // from implementation + Assert.AreEqual(transceiver1.NegotiatedDirection, null); + Assert.AreEqual(pc1_, transceiver1.PeerConnection); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); + + // Wait for local SDP re-negotiation event on #1. + // This will not create an offer, since we're not connected yet. + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(60.0))); + renegotiationEvent1_.Reset(); + + // Connect + await DoNegotiationStartFrom(pc1_); + + // Note: use manual list instead of Enum.GetValues() to control order, and not + // get Inactive first (which is the current value, so wouldn't make any change). + var desired = new List { + Transceiver.Direction.SendOnly, Transceiver.Direction.SendReceive, + Transceiver.Direction.ReceiveOnly, Transceiver.Direction.Inactive }; + var negotiated = new List { + Transceiver.Direction.SendOnly, Transceiver.Direction.SendOnly, + Transceiver.Direction.Inactive, Transceiver.Direction.Inactive }; + for (int i = 0; i < desired.Count; ++i) + { + var direction = desired[i]; + + // Change flow direction + renegotiationEvent1_.Reset(); + transceiver1.DesiredDirection = direction; + Assert.AreEqual(transceiver1.DesiredDirection, direction); + + // Wait for local SDP re-negotiation event on #1. + Assert.True(renegotiationEvent1_.Wait(TimeSpan.FromSeconds(10.0))); + renegotiationEvent1_.Reset(); + + // Renegotiate + await DoNegotiationStartFrom(pc1_); + + // Observe the new negotiated direction + Assert.AreEqual(transceiver1.DesiredDirection, direction); + Assert.AreEqual(transceiver1.NegotiatedDirection, negotiated[i]); + } + } + + [Test(Description = "Check that the transceiver stream IDs are correctly broadcast to the remote peer.")] + public async Task StreamIDs() + { + // This test use manual offers + suspendOffer1_ = true; + + // Create video transceiver on #1. This triggers a renegotiation needed event. + string name1 = "video_feed"; + var initSettings = new TransceiverInitSettings + { + Name = name1, + InitialDesiredDirection = Transceiver.Direction.SendOnly, + StreamIDs = new List { "id1", "id2" } + }; + var transceiver1 = pc1_.AddTransceiver(MediaKind, initSettings); + Assert.NotNull(transceiver1); + // Names are equal only because the transceiver was created by the local peer + Assert.AreEqual(name1, transceiver1.Name); + Assert.AreEqual(2, transceiver1.StreamIDs.Length); + Assert.AreEqual("id1", transceiver1.StreamIDs[0]); + Assert.AreEqual("id2", transceiver1.StreamIDs[1]); + Assert.AreEqual(transceiver1.DesiredDirection, Transceiver.Direction.SendOnly); + Assert.AreEqual(transceiver1.NegotiatedDirection, null); + Assert.AreEqual(pc1_, transceiver1.PeerConnection); + Assert.IsTrue(pc1_.Transceivers.Contains(transceiver1)); + + // Connect + await DoNegotiationStartFrom(pc1_); + + // Find the remote transceiver + Assert.AreEqual(1, pc2_.Transceivers.Count); + var transceiver2 = pc2_.Transceivers[0]; + Assert.NotNull(transceiver2); + + // Check stream IDs were associated + Assert.AreEqual(2, transceiver2.StreamIDs.Length); + Assert.AreEqual("id1", transceiver2.StreamIDs[0]); + Assert.AreEqual("id2", transceiver2.StreamIDs[1]); + } + + [Test(Description = "#179 - Ensure AddTransceiver(mediaKind, null) works.")] + public void AddTransceiver_Null() + { + var tr = pc1_.AddTransceiver(MediaKind, null); + Assert.IsNotNull(tr); + } + + [Test(Description = "#179 - Ensure AddTransceiver(mediaKind, default) works.")] + public void AddTransceiver_Default() + { + var settings = new TransceiverInitSettings(); + var tr = pc1_.AddTransceiver(MediaKind, settings); + Assert.IsNotNull(tr); + } + + [Test] + public void AddTransceiver_InvalidName() + { + var settings = new TransceiverInitSettings(); + settings.Name = "invalid name"; + Transceiver tr = null; + Assert.Throws(() => { tr = pc1_.AddTransceiver(MediaKind, settings); }); + Assert.IsNull(tr); + } + } +}