Add multi-track support to C# library via transceiver API

libs/Microsoft.MixedReality.WebRTC.Native/src/peer_connection.cpp

@@ -951,7 +951,8 @@ void PeerConnectionImpl::RemoveAllDataChannels() noexcept {
 
     // Invoke the DataChannelRemoved callback on the wrapper if any
     if (removed_cb) {
-      mrsDataChannelHandle data_native_handle = (void*)&data_channel;
+      DataChannel* const dc = data_channel.get();
+      mrsDataChannelHandle data_native_handle = (void*)dc;
       removed_cb(data_native_handle);
     }
 
@@ -1121,18 +1122,18 @@ void PeerConnectionImpl::Close() noexcept {
     for (auto&& transceiver : transceivers_) {
       if (auto remote_track = transceiver->GetRemoteTrack()) {
         if (remote_track->GetKind() == mrsTrackKind::kAudioTrack) {
-          auto* const audio_track =
-              static_cast<RemoteAudioTrack*>(remote_track);
+          RefPtr<RemoteAudioTrack> audio_track(
+              static_cast<RemoteAudioTrack*>(remote_track));  // keep alive
           audio_track->OnTrackRemoved(*this);
           if (audio_cb) {
-            audio_cb(audio_track, transceiver.get());
+            audio_cb(audio_track.get(), transceiver.get());
           }
         } else if (remote_track->GetKind() == mrsTrackKind::kVideoTrack) {
-          auto* const video_track =
-              static_cast<RemoteVideoTrack*>(remote_track);
+          RefPtr<RemoteVideoTrack> video_track(
+              static_cast<RemoteVideoTrack*>(remote_track));  // keep alive
           video_track->OnTrackRemoved(*this);
           if (video_cb) {
-            video_cb(video_track, transceiver.get());
+            video_cb(video_track.get(), transceiver.get());
           }
         }
       }
@@ -1628,8 +1629,8 @@ PeerConnectionImpl::GetOrCreateTransceiverForNewRemoteTrack(
     int mline_index = -1;
     std::string name;
     std::vector<std::string> stream_ids;
-    if (!ExtractTransceiverInfoFromReceiverPlanB(receiver, mline_index,
-                                                        name, stream_ids)) {
+    if (!ExtractTransceiverInfoFromReceiverPlanB(receiver, mline_index, name,
+                                                 stream_ids)) {
       return Error(
           Result::kUnknownError,
           "Failed to associate RTP receiver with Plan B emulated mline index "

libs/Microsoft.MixedReality.WebRTC/DataChannel.cs

@@ -2,36 +2,42 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Runtime.InteropServices;
+using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 using Microsoft.MixedReality.WebRTC.Tracing;
 
 namespace Microsoft.MixedReality.WebRTC
 {
     /// <summary>
     /// Encapsulates a data channel of a peer connection.
-    /// 
-    /// A data channel is a pipe allowing to send and receive arbitrary data to the
+    ///
+    /// A data channel is a "pipe" allowing to send and receive arbitrary data to the
     /// remote peer. Data channels are based on DTLS-SRTP, and are therefore secure (encrypted).
     /// Exact security guarantees are provided by the underlying WebRTC core implementation
     /// and the WebRTC standard itself.
-    /// 
+    ///
     /// https://tools.ietf.org/wg/rtcweb/
-    /// 
-    /// An instance of <see cref="DataChannel"/> is created by calling <see cref="PeerConnection.AddDataChannelAsync(string,bool,bool)"/>
-    /// or one of its variants. <see cref="DataChannel"/> cannot be instantiated directly.
+    /// https://www.w3.org/TR/webrtc/
+    ///
+    /// An instance of <see cref="DataChannel"/> is created either by manually calling
+    /// <see cref="PeerConnection.AddDataChannelAsync(string,bool,bool)"/> or one of its variants,
+    /// or automatically by the implementation when a new data channel is created in-band by the
+    /// remote peer (<see cref="PeerConnection.DataChannelAdded"/>).
+    /// <see cref="DataChannel"/> cannot be instantiated directly.
     /// </summary>
-    public class DataChannel : IDisposable
+    /// <seealso cref="PeerConnection.AddDataChannelAsync(string, bool, bool)"/>
+    /// <seealso cref="PeerConnection.AddDataChannelAsync(ushort, string, bool, bool)"/>
+    /// <seealso cref="PeerConnection.DataChannelAdded"/>
+    public class DataChannel
     {
         /// <summary>
-        /// Connecting state of a data channel, when adding it to a peer connection
-        /// or removing it from a peer connection.
+        /// Connection state of a data channel.
         /// </summary>
         public enum ChannelState
         {
             /// <summary>
             /// The data channel has just been created, and negotiating is underway to establish
-            /// a track between the peers.
+            /// a link between the peers. The data channel cannot be used to send/receive yet.
             /// </summary>
             Connecting = 0,
 
@@ -60,13 +66,19 @@ public enum ChannelState
         /// <param name="limit">Maximum buffering size, in bytes.</param>
         public delegate void BufferingChangedDelegate(ulong previous, ulong current, ulong limit);
 
-        /// <value>The <see cref="PeerConnection"/> object this data channel was created from.</value>
+        /// <summary>
+        /// The <see cref="PeerConnection"/> object this data channel was created from and is attached to.
+        /// </summary>
         public PeerConnection PeerConnection { get; }
 
-        /// <value>The unique identifier of the data channel in the current connection.</value>
+        /// <summary>
+        /// The unique identifier of the data channel in the current connection.
+        /// </summary>
         public int ID { get; }
 
-        /// <value>The data channel name in the current connection.</value>
+        /// <summary>
+        /// The data channel name in the current connection.
+        /// </summary>
         public string Label { get; }
 
         /// <summary>
@@ -89,22 +101,23 @@ public enum ChannelState
         public bool Reliable { get; }
 
         /// <summary>
-        /// The channel connection state represents the connection status when creating or closing the
-        /// data channel. Changes to this state are notified via the <see cref="StateChanged"/> event.
+        /// The channel connection state represents the connection status.
+        /// Changes to this state are notified via the <see cref="StateChanged"/> event.
         /// </summary>
         /// <value>The channel connection state.</value>
         /// <seealso cref="StateChanged"/>
-        public ChannelState State { get; private set; }
+        public ChannelState State { get; internal set; }
 
         /// <summary>
-        /// Event fired when the data channel state changes, as reported by <see cref="State"/>.
+        /// Event triggered when the data channel state changes.
+        /// The new state is available in <see cref="State"/>.
         /// </summary>
         /// <seealso cref="State"/>
         public event Action StateChanged;
 
         /// <summary>
-        /// Event fired when the data channel buffering changes. Monitor this to ensure calls to
-        /// <see cref="SendMessage(byte[])"/> do not fail. Internally the data channel contains
+        /// Event triggered when the data channel buffering changes. Users should monitor this to ensure
+        /// calls to <see cref="SendMessage(byte[])"/> do not fail. Internally the data channel contains
         /// a buffer of messages to send that could not be sent immediately, for example due to
         /// congestion control. Once this buffer is full, any further call to <see cref="SendMessage(byte[])"/>
         /// will fail until some mesages are processed and removed to make space.
@@ -113,26 +126,31 @@ public enum ChannelState
         public event BufferingChangedDelegate BufferingChanged;
 
         /// <summary>
-        /// Event fires when a message is received through the data channel.
+        /// Event triggered when a message is received through the data channel.
         /// </summary>
         /// <seealso cref="SendMessage(byte[])"/>
         public event Action<byte[]> MessageReceived;
 
         /// <summary>
-        /// GC handle keeping the internal delegates alive while they are registered
+        /// Reference (GC handle) keeping the internal delegates alive while they are registered
         /// as callbacks with the native code.
         /// </summary>
-        private GCHandle _handle;
+        /// <seealso cref="Utils.MakeWrapperRef(object)"/>
+        private IntPtr _argsRef;
 
         /// <summary>
-        /// Handle to the native object this wrapper is associated with.
+        /// Handle to the native DataChannel object.
         /// </summary>
-        internal IntPtr _interopHandle = IntPtr.Zero;
+        /// <remarks>
+        /// In native land this is a <code>mrsDataChannelHandle</code>.
+        /// </remarks>
+        internal IntPtr _nativeHandle = IntPtr.Zero;
 
-        internal DataChannel(PeerConnection peer, GCHandle handle,
-            int id, string label, bool ordered, bool reliable)
+        internal DataChannel(IntPtr nativeHandle, PeerConnection peer, IntPtr argsRef, int id, string label, bool ordered, bool reliable)
         {
-            _handle = handle;
+            Debug.Assert(nativeHandle != IntPtr.Zero);
+            _nativeHandle = nativeHandle;
+            _argsRef = argsRef;
             PeerConnection = peer;
             ID = id;
             Label = label;
@@ -142,25 +160,14 @@ internal DataChannel(PeerConnection peer, GCHandle handle,
         }
 
         /// <summary>
-        /// Finalizer to ensure the data track is removed from the peer connection
-        /// and the managed resources are cleaned-up.
+        /// Dispose of the native data channel. Invoked by its owner (<see cref="PeerConnection"/>).
         /// </summary>
-        ~DataChannel()
+        internal void DestroyNative()
         {
-            Dispose();
-        }
-
-        /// <summary>
-        /// Remove the data track from the peer connection and destroy it.
-        /// </summary>
-        public void Dispose()
-        {
-            State = ChannelState.Closing;
-            PeerConnection.RemoveDataChannel(_interopHandle);
-            _interopHandle = IntPtr.Zero;
+            _nativeHandle = IntPtr.Zero;
             State = ChannelState.Closed;
-            _handle.Free();
-            GC.SuppressFinalize(this);
+            Utils.ReleaseWrapperRef(_argsRef);
+            _argsRef = IntPtr.Zero;
         }
 
         /// <summary>
@@ -169,15 +176,21 @@ public void Dispose()
         /// The internal buffering is monitored via the <see cref="BufferingChanged"/> event.
         /// </summary>
         /// <param name="message">The message to send to the remote peer.</param>
-        /// <exception xref="InvalidOperationException">The native data channel is not initialized.</exception>
-        /// <exception xref="Exception">The internal buffer is full.</exception>
+        /// <exception xref="System.InvalidOperationException">The native data channel is not initialized.</exception>
+        /// <exception xref="System.Exception">The internal buffer is full.</exception>
+        /// <exception cref="DataChannelNotOpenException">The data channel is not open yet.</exception>
         /// <seealso cref="PeerConnection.InitializeAsync"/>
         /// <seealso cref="PeerConnection.Initialized"/>
         /// <seealso cref="BufferingChanged"/>
         public void SendMessage(byte[] message)
         {
             MainEventSource.Log.DataChannelSendMessage(ID, message.Length);
-            uint res = DataChannelInterop.DataChannel_SendMessage(_interopHandle, message, (ulong)message.LongLength);
+            // Check channel state before doing a P/Invoke call which would anyway fail
+            if (State != ChannelState.Open)
+            {
+                throw new DataChannelNotOpenException();
+            }
+            uint res = DataChannelInterop.DataChannel_SendMessage(_nativeHandle, message, (ulong)message.LongLength);
             Utils.ThrowOnErrorCode(res);
         }
 

libs/Microsoft.MixedReality.WebRTC/Exceptions.cs

@@ -58,4 +58,32 @@ public InvalidInteropNativeHandleException(string message, Exception inner)
         {
         }
     }
+
+    /// <summary>
+    /// Exception thrown when trying to use a data channel that is not open.
+    ///
+    /// The user should listen to the <see cref="DataChannel.StateChanged"/> event until the
+    /// <see cref="DataChannel.State"/> property is <see cref="DataChannel.ChannelState.Open"/>
+    /// before trying to send some message with <see cref="DataChannel.SendMessage(byte[])"/>.
+    /// </summary>
+    public class DataChannelNotOpenException : Exception
+    {
+        /// <inheritdoc/>
+        public DataChannelNotOpenException()
+            : base("Data channel is not open yet, cannot send message. Wait for the StateChanged event until State is Open.")
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message)
+            : base(message)
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message, Exception inner)
+            : base(message, inner)
+        {
+        }
+    }
 }

libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 
@@ -81,10 +82,14 @@ public void CompleteRequest(in Argb32VideoFrame frame)
     public class ExternalVideoTrackSource : IDisposable
     {
         /// <summary>
-        /// Once the external video track source is attached to some video track(s), this returns the peer connection
-        /// the video track(s) are part of. Otherwise this returns <c>null</c>.
+        /// A name for the external video track source, used for logging and debugging.
         /// </summary>
-        public PeerConnection PeerConnection { get; private set; } = null;
+        public string Name { get; set; } = string.Empty;
+
+        /// <summary>
+        /// List of local video tracks this source is providing raw video frames to.
+        /// </summary>
+        public List<LocalVideoTrack> Tracks { get; } = new List<LocalVideoTrack>();
 
         /// <summary>
         /// Handle to the native ExternalVideoTrackSource object.
@@ -164,10 +169,6 @@ public void Dispose()
                 return;
             }
 
-            // Remove the tracks associated with this source from the peer connection, if any
-            PeerConnection?.RemoveLocalVideoTracksFromSource(this);
-            Debug.Assert(PeerConnection == null); // see OnTracksRemovedFromSource
-
             // Unregister and release the track callbacks
             ExternalVideoTrackSourceInterop.ExternalVideoTrackSource_Shutdown(_nativeHandle);
             Utils.ReleaseWrapperRef(_frameRequestCallbackArgsHandle);
@@ -177,22 +178,43 @@ public void Dispose()
             _nativeHandle.Dispose();
         }
 
-        internal void OnTracksAddedToSource(PeerConnection newConnection)
+        internal void OnTrackAddedToSource(LocalVideoTrack track)
+        {
+            Debug.Assert(!_nativeHandle.IsClosed);
+            Debug.Assert(!Tracks.Contains(track));
+            Tracks.Add(track);
+        }
+
+        internal void OnTrackRemovedFromSource(LocalVideoTrack track)
         {
-            Debug.Assert(PeerConnection == null);
             Debug.Assert(!_nativeHandle.IsClosed);
-            PeerConnection = newConnection;
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = newConnection;
+            bool removed = Tracks.Remove(track);
+            Debug.Assert(removed);
         }
 
-        internal void OnTracksRemovedFromSource(PeerConnection previousConnection)
+        internal void OnTracksRemovedFromSource(List<LocalVideoTrack> tracks)
         {
-            Debug.Assert(PeerConnection == previousConnection);
             Debug.Assert(!_nativeHandle.IsClosed);
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = null;
-            PeerConnection = null;
+            var remainingTracks = new List<LocalVideoTrack>();
+            foreach (var track in tracks)
+            {
+                if (track.Source == this)
+                {
+                    bool removed = Tracks.Remove(track);
+                    Debug.Assert(removed);
+                }
+                else
+                {
+                    remainingTracks.Add(track);
+                }
+            }
+            tracks = remainingTracks;
+        }
+
+        /// <inheritdoc/>
+        public override string ToString()
+        {
+            return $"(ExternalVideoTrackSource)\"{Name}\"";
         }
     }
 }

libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.MixedReality.WebRTC.Interop
+{
+    internal class AudioTrackInterop
+    {
+        #region Native callbacks
+
+        // The callbacks below ideally would use 'in', but that generates an error with .NET Native:
+        // "error : ILT0021: Could not resolve method 'EETypeRva:0x--------'".
+        // So instead use 'ref' to ensure the signature is compatible with the C++ const& signature.
+
+        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)]
+        public delegate void AudioFrameUnmanagedCallback(IntPtr userData, ref AudioFrame frame);
+
+        #endregion
+    }
+}