Add multi-track support to C# library via transceiver API

libs/Microsoft.MixedReality.WebRTC/DataChannel.cs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Runtime.InteropServices;
+using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 using Microsoft.MixedReality.WebRTC.Tracing;
 
@@ -97,13 +97,13 @@ public enum ChannelState
         public ChannelState State { get; private set; }
 
         /// <summary>
-        /// Event fired when the data channel state changes, as reported by <see cref="State"/>.
+        /// Event invoked when the data channel state changes, as reported by <see cref="State"/>.
         /// </summary>
         /// <seealso cref="State"/>
         public event Action StateChanged;
 
         /// <summary>
-        /// Event fired when the data channel buffering changes. Monitor this to ensure calls to
+        /// Event invoked when the data channel buffering changes. Monitor this to ensure calls to
         /// <see cref="SendMessage(byte[])"/> do not fail. Internally the data channel contains
         /// a buffer of messages to send that could not be sent immediately, for example due to
         /// congestion control. Once this buffer is full, any further call to <see cref="SendMessage(byte[])"/>
@@ -122,17 +122,21 @@ public enum ChannelState
         /// GC handle keeping the internal delegates alive while they are registered
         /// as callbacks with the native code.
         /// </summary>
-        private GCHandle _handle;
+        private IntPtr _argsRef;
 
         /// <summary>
-        /// Handle to the native object this wrapper is associated with.
+        /// Handle to the native DataChannel object.
         /// </summary>
-        internal IntPtr _interopHandle = IntPtr.Zero;
+        /// <remarks>
+        /// In native land this is a <code>mrsDataChannelHandle</code>.
+        /// </remarks>
+        internal IntPtr _nativeHandle = IntPtr.Zero;
 
-        internal DataChannel(PeerConnection peer, GCHandle handle,
-            int id, string label, bool ordered, bool reliable)
+        internal DataChannel(IntPtr nativeHandle, PeerConnection peer, IntPtr argsRef, int id, string label, bool ordered, bool reliable)
         {
-            _handle = handle;
+            Debug.Assert(nativeHandle != IntPtr.Zero);
+            _nativeHandle = nativeHandle;
+            _argsRef = argsRef;
             PeerConnection = peer;
             ID = id;
             Label = label;
@@ -156,10 +160,11 @@ internal DataChannel(PeerConnection peer, GCHandle handle,
         public void Dispose()
         {
             State = ChannelState.Closing;
-            PeerConnection.RemoveDataChannel(_interopHandle);
-            _interopHandle = IntPtr.Zero;
+            PeerConnection.RemoveDataChannel(_nativeHandle);
+            _nativeHandle = IntPtr.Zero;
             State = ChannelState.Closed;
-            _handle.Free();
+            Utils.ReleaseWrapperRef(_argsRef);
+            _argsRef = IntPtr.Zero;
             GC.SuppressFinalize(this);
         }
 
@@ -169,15 +174,21 @@ public void Dispose()
         /// The internal buffering is monitored via the <see cref="BufferingChanged"/> event.
         /// </summary>
         /// <param name="message">The message to send to the remote peer.</param>
-        /// <exception xref="InvalidOperationException">The native data channel is not initialized.</exception>
-        /// <exception xref="Exception">The internal buffer is full.</exception>
+        /// <exception xref="System.InvalidOperationException">The native data channel is not initialized.</exception>
+        /// <exception xref="System.Exception">The internal buffer is full.</exception>
+        /// <exception cref="DataChannelNotOpenException">The data channel is not open yet.</exception>
         /// <seealso cref="PeerConnection.InitializeAsync"/>
         /// <seealso cref="PeerConnection.Initialized"/>
         /// <seealso cref="BufferingChanged"/>
         public void SendMessage(byte[] message)
         {
             MainEventSource.Log.DataChannelSendMessage(ID, message.Length);
-            uint res = DataChannelInterop.DataChannel_SendMessage(_interopHandle, message, (ulong)message.LongLength);
+            // Check channel state before doing a P/Invoke call which would anyway fail
+            if (State != ChannelState.Open)
+            {
+                throw new DataChannelNotOpenException();
+            }
+            uint res = DataChannelInterop.DataChannel_SendMessage(_nativeHandle, message, (ulong)message.LongLength);
             Utils.ThrowOnErrorCode(res);
         }
 

libs/Microsoft.MixedReality.WebRTC/Exceptions.cs

@@ -58,4 +58,32 @@ public InvalidInteropNativeHandleException(string message, Exception inner)
         {
         }
     }
+
+    /// <summary>
+    /// Exception thrown when trying to use a data channel that is not open.
+    ///
+    /// The user should listen to the <see cref="DataChannel.StateChanged"/> event until the
+    /// <see cref="DataChannel.State"/> property is <see cref="DataChannel.ChannelState.Open"/>
+    /// before trying to send some message with <see cref="DataChannel.SendMessage(byte[])"/>.
+    /// </summary>
+    public class DataChannelNotOpenException : Exception
+    {
+        /// <inheritdoc/>
+        public DataChannelNotOpenException()
+            : base("Data channel is not open yet, cannot send message. Wait for the StateChanged event until State is Open.")
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message)
+            : base(message)
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message, Exception inner)
+            : base(message, inner)
+        {
+        }
+    }
 }

libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 
@@ -81,10 +82,14 @@ public void CompleteRequest(in Argb32VideoFrame frame)
     public class ExternalVideoTrackSource : IDisposable
     {
         /// <summary>
-        /// Once the external video track source is attached to some video track(s), this returns the peer connection
-        /// the video track(s) are part of. Otherwise this returns <c>null</c>.
+        /// A name for the external video track source, used for logging and debugging.
         /// </summary>
-        public PeerConnection PeerConnection { get; private set; } = null;
+        public string Name { get; set; } = string.Empty;
+
+        /// <summary>
+        /// List of local video tracks this source is providing raw video frames to.
+        /// </summary>
+        public List<LocalVideoTrack> Tracks { get; } = new List<LocalVideoTrack>();
 
         /// <summary>
         /// Handle to the native ExternalVideoTrackSource object.
@@ -164,10 +169,6 @@ public void Dispose()
                 return;
             }
 
-            // Remove the tracks associated with this source from the peer connection, if any
-            PeerConnection?.RemoveLocalVideoTracksFromSource(this);
-            Debug.Assert(PeerConnection == null); // see OnTracksRemovedFromSource
-
             // Unregister and release the track callbacks
             ExternalVideoTrackSourceInterop.ExternalVideoTrackSource_Shutdown(_nativeHandle);
             Utils.ReleaseWrapperRef(_frameRequestCallbackArgsHandle);
@@ -177,22 +178,43 @@ public void Dispose()
             _nativeHandle.Dispose();
         }
 
-        internal void OnTracksAddedToSource(PeerConnection newConnection)
+        internal void OnTrackAddedToSource(LocalVideoTrack track)
+        {
+            Debug.Assert(!_nativeHandle.IsClosed);
+            Debug.Assert(!Tracks.Contains(track));
+            Tracks.Add(track);
+        }
+
+        internal void OnTrackRemovedFromSource(LocalVideoTrack track)
         {
-            Debug.Assert(PeerConnection == null);
             Debug.Assert(!_nativeHandle.IsClosed);
-            PeerConnection = newConnection;
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = newConnection;
+            bool removed = Tracks.Remove(track);
+            Debug.Assert(removed);
         }
 
-        internal void OnTracksRemovedFromSource(PeerConnection previousConnection)
+        internal void OnTracksRemovedFromSource(List<LocalVideoTrack> tracks)
         {
-            Debug.Assert(PeerConnection == previousConnection);
             Debug.Assert(!_nativeHandle.IsClosed);
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = null;
-            PeerConnection = null;
+            var remainingTracks = new List<LocalVideoTrack>();
+            foreach (var track in tracks)
+            {
+                if (track.Source == this)
+                {
+                    bool removed = Tracks.Remove(track);
+                    Debug.Assert(removed);
+                }
+                else
+                {
+                    remainingTracks.Add(track);
+                }
+            }
+            tracks = remainingTracks;
+        }
+
+        /// <inheritdoc/>
+        public override string ToString()
+        {
+            return $"(ExternalVideoTrackSource)\"{Name}\"";
         }
     }
 }

libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.MixedReality.WebRTC.Interop
+{
+    internal class AudioTrackInterop
+    {
+        #region Native callbacks
+
+        // The callbacks below ideally would use 'in', but that generates an error with .NET Native:
+        // "error : ILT0021: Could not resolve method 'EETypeRva:0x--------'".
+        // So instead use 'ref' to ensure the signature is compatible with the C++ const& signature.
+
+        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)]
+        public delegate void AudioFrameUnmanagedCallback(IntPtr userData, ref AudioFrame frame);
+
+        #endregion
+    }
+}

libs/Microsoft.MixedReality.WebRTC/Interop/DataChannelInterop.cs

@@ -11,6 +11,18 @@ internal class DataChannelInterop
     {
         #region Native functions
 
+        [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi,
+            EntryPoint = "mrsDataChannelSetUserData")]
+        public static unsafe extern void DataChannel_SetUserData(IntPtr handle, IntPtr userData);
+
+        [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi,
+            EntryPoint = "mrsDataChannelGetUserData")]
+        public static unsafe extern IntPtr DataChannel_GetUserData(IntPtr handle);
+
+        [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi,
+            EntryPoint = "mrsDataChannelRegisterCallbacks")]
+        public static extern void DataChannel_RegisterCallbacks(IntPtr dataChannelHandle, ref Callbacks callbacks);
+
         [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi,
             EntryPoint = "mrsDataChannelSendMessage")]
         public static extern uint DataChannel_SendMessage(IntPtr dataChannelHandle, byte[] data, ulong size);
@@ -24,8 +36,8 @@ internal class DataChannelInterop
         public ref struct CreateConfig
         {
             public int id;
-            public string label;
             public uint flags;
+            public string label;
         }
 
         [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
@@ -44,10 +56,6 @@ public ref struct Callbacks
 
         #region Native callbacks
 
-        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)]
-        public delegate IntPtr CreateObjectDelegate(IntPtr peer, CreateConfig config,
-            out Callbacks callbacks);
-
         [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)]
         public delegate void MessageCallback(IntPtr userData, IntPtr data, ulong size);
 
@@ -71,15 +79,6 @@ public class CallbackArgs
             public StateCallback StateCallback;
         }
 
-        [MonoPInvokeCallback(typeof(CreateObjectDelegate))]
-        public static IntPtr DataChannelCreateObjectCallback(IntPtr peer, CreateConfig config,
-            out Callbacks callbacks)
-        {
-            var peerWrapper = Utils.ToWrapper<PeerConnection>(peer);
-            var dataChannelWrapper = CreateWrapper(peerWrapper, config, out callbacks);
-            return Utils.MakeWrapperRef(dataChannelWrapper);
-        }
-
         [MonoPInvokeCallback(typeof(MessageCallback))]
         public static void DataChannelMessageCallback(IntPtr userData, IntPtr data, ulong size)
         {
@@ -106,7 +105,7 @@ public static void DataChannelStateCallback(IntPtr userData, int state, int id)
 
         #region Utilities
 
-        public static DataChannel CreateWrapper(PeerConnection parent, CreateConfig config, out Callbacks callbacks)
+        public static DataChannel CreateWrapper(PeerConnection parent, in PeerConnectionInterop.DataChannelAddedInfo info)
         {
             // Create the callback args for the data channel
             var args = new CallbackArgs()
@@ -119,37 +118,33 @@ public static DataChannel CreateWrapper(PeerConnection parent, CreateConfig conf
             };
 
             // Pin the args to pin the delegates while they're registered with the native code
-            var handle = GCHandle.Alloc(args, GCHandleType.Normal);
-            IntPtr userData = GCHandle.ToIntPtr(handle);
+            IntPtr argsRef = Utils.MakeWrapperRef(args);
 
-            // Create a new data channel. It will hold the lock for its args while alive.
-            bool ordered = (config.flags & 0x1) != 0;
-            bool reliable = (config.flags & 0x2) != 0;
-            var dataChannel = new DataChannel(parent, handle, config.id, config.label, ordered, reliable);
+            // Create a new data channel wrapper. It will hold the lock for its args while alive.
+            bool ordered = (info.flags & 0x1) != 0;
+            bool reliable = (info.flags & 0x2) != 0;
+            var dataChannel = new DataChannel(info.dataChannelHandle, parent, argsRef, info.id, info.label, ordered, reliable);
             args.DataChannel = dataChannel;
 
-            // Fill out the callbacks
-            callbacks = new Callbacks()
+            // Assign a reference to it inside the UserData of the native object so it can be retrieved whenever needed
+            IntPtr wrapperRef = Utils.MakeWrapperRef(dataChannel);
+            DataChannel_SetUserData(info.dataChannelHandle, wrapperRef);
+
+            // Register the callbacks
+            var callbacks = new Callbacks()
             {
                 messageCallback = args.MessageCallback,
-                messageUserData = userData,
+                messageUserData = argsRef,
                 bufferingCallback = args.BufferingCallback,
-                bufferingUserData = userData,
+                bufferingUserData = argsRef,
                 stateCallback = args.StateCallback,
-                stateUserData = userData
+                stateUserData = argsRef
             };
+            DataChannel_RegisterCallbacks(info.dataChannelHandle, ref callbacks);
 
             return dataChannel;
         }
 
-        public static void SetHandle(DataChannel dataChannel, IntPtr handle)
-        {
-            Debug.Assert(handle != IntPtr.Zero);
-            // Either first-time assign or no-op (assign same value again)
-            Debug.Assert((dataChannel._interopHandle == IntPtr.Zero) || (dataChannel._interopHandle == handle));
-            dataChannel._interopHandle = handle;
-        }
-
         #endregion
     }
 }

libs/Microsoft.MixedReality.WebRTC/Interop/ExternalVideoTrackSourceInterop.cs

@@ -100,7 +100,6 @@ public static void RequestArgb32VideoFrameFromExternalSourceCallback(IntPtr user
 
         public class VideoFrameRequestCallbackArgs
         {
-            public PeerConnection Peer;
             public ExternalVideoTrackSource Source;
         }
 
@@ -166,7 +165,6 @@ public static ExternalVideoTrackSource CreateExternalVideoTrackSourceFromI420ACa
             // Create some static callback args which keep the sourceDelegate alive
             var args = new I420AVideoFrameRequestCallbackArgs
             {
-                Peer = null, // set when track added
                 Source = null, // set below
                 FrameRequestCallback = frameRequestCallback,
                 // This wraps the method into a temporary System.Delegate object, which is then assigned to
@@ -206,7 +204,6 @@ public static ExternalVideoTrackSource CreateExternalVideoTrackSourceFromArgb32C
             // Create some static callback args which keep the sourceDelegate alive
             var args = new Argb32VideoFrameRequestCallbackArgs
             {
-                Peer = null, // set when track added
                 Source = null, // set below
                 FrameRequestCallback = frameRequestCallback,
                 // This wraps the method into a temporary System.Delegate object, which is then assigned to