Add multi-track support to C# library via transceiver API

libs/Microsoft.MixedReality.WebRTC/AudioTransceiver.cs

@@ -0,0 +1,146 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.MixedReality.WebRTC.Interop;
+using System;
+using System.Diagnostics;
+
+namespace Microsoft.MixedReality.WebRTC
+{
+    /// <summary>
+    /// Transceiver for audio tracks.
+    /// </summary>
+    public class AudioTransceiver : Transceiver
+    {
+        /// <summary>
+        /// Local audio track associated with the transceiver and sending some audio to the remote peer.
+        /// This may be <c>null</c> if the transceiver is currently only receiving, or is inactive.
+        /// </summary>
+        public LocalAudioTrack LocalTrack
+        {
+            get { return _localTrack; }
+            set { SetLocalTrack(value); }
+        }
+
+        /// <summary>
+        /// Remote audio track associated with the transceiver and receiving some audio from the remote peer.
+        /// This property is updated when the transceiver negotiates a new media direction. If the new direction
+        /// allows receiving some audio, a new <see cref="RemoteVideoTrack"/> is created and assigned to the
+        /// property. Otherwise the property is cleared to <c>null</c>.
+        /// </summary>
+        public RemoteAudioTrack RemoteTrack { get; private set; } = null;
+
+        /// <summary>
+        /// Backing field for <see cref="LocalTrack"/>.
+        /// </summary>
+        private LocalAudioTrack _localTrack = null;
+
+        internal AudioTransceiver(IntPtr handle, PeerConnection peerConnection, int mlineIndex, string name, string[] streamIDs, Direction initialDesiredDirection)
+            : base(handle, MediaKind.Audio, peerConnection, mlineIndex, name, streamIDs, initialDesiredDirection)
+        {
+        }
+
+        /// <summary>
+        /// Change the local audio track sending data to the remote peer.
+        /// 
+        /// This detaches the previous local audio track if any, and attaches the new one instead.
+        /// Note that the transceiver will only send some audio data to the remote peer if its
+        /// negotiated direction includes sending some data and it has an attached local track to
+        /// produce this data.
+        /// 
+        /// This change is transparent to the session, and does not trigger any renegotiation.
+        /// </summary>
+        /// <param name="track">The new local audio track attached to the transceiver, and used to
+        /// produce audio data to send to the remote peer if the transceiver is sending.
+        /// Passing <c>null</c> is allowed, and will detach the current track if any.</param>
+        public void SetLocalTrack(LocalAudioTrack track)
+        {
+            if (track == _localTrack)
+            {
+                return;
+            }
+
+            if (track != null)
+            {
+                if ((track.PeerConnection != null) && (track.PeerConnection != PeerConnection))
+                {
+                    throw new InvalidOperationException($"Cannot set track {track} of peer connection {track.PeerConnection} on audio transceiver {this} of different peer connection {PeerConnection}.");
+                }
+                var res = TransceiverInterop.Transceiver_SetLocalAudioTrack(_nativeHandle, track._nativeHandle);
+                Utils.ThrowOnErrorCode(res);
+            }
+            else
+            {
+                // Note: Cannot pass null for SafeHandle parameter value (ArgumentNullException)
+                var res = TransceiverInterop.Transceiver_SetLocalAudioTrack(_nativeHandle, new LocalAudioTrackHandle());
+                Utils.ThrowOnErrorCode(res);
+            }
+
+            // Capture peer connection; it gets reset during track manipulation below
+            var peerConnection = PeerConnection;
+
+            // Remove old track
+            if (_localTrack != null)
+            {
+                _localTrack.OnTrackRemoved(peerConnection);
+            }
+            Debug.Assert(_localTrack == null);
+
+            // Add new track
+            if (track != null)
+            {
+                track.OnTrackAdded(peerConnection, this);
+                Debug.Assert(track == _localTrack);
+                Debug.Assert(_localTrack.PeerConnection == PeerConnection);
+                Debug.Assert(_localTrack.Transceiver == this);
+                Debug.Assert(_localTrack.Transceiver.LocalTrack == _localTrack);
+            }
+        }
+
+        /// <inheritdoc/>
+        protected override void OnLocalTrackMuteChanged(bool muted)
+        {
+            LocalTrack?.OnMute(muted);
+        }
+
+        /// <inheritdoc/>
+        protected override void OnRemoteTrackMuteChanged(bool muted)
+        {
+            RemoteTrack?.OnMute(muted);
+        }
+
+        internal void OnLocalTrackAdded(LocalAudioTrack track)
+        {
+            Debug.Assert(_localTrack == null);
+            _localTrack = track;
+            PeerConnection.OnLocalTrackAdded(track);
+        }
+
+        internal void OnLocalTrackRemoved(LocalAudioTrack track)
+        {
+            Debug.Assert(_localTrack == track);
+            _localTrack = null;
+            PeerConnection.OnLocalTrackRemoved(track);
+        }
+
+        internal void OnRemoteTrackAdded(RemoteAudioTrack track)
+        {
+            Debug.Assert(RemoteTrack == null);
+            RemoteTrack = track;
+            PeerConnection.OnRemoteTrackAdded(track);
+        }
+
+        internal void OnRemoteTrackRemoved(RemoteAudioTrack track)
+        {
+            Debug.Assert(RemoteTrack == track);
+            RemoteTrack = null;
+            PeerConnection.OnRemoteTrackRemoved(track);
+        }
+
+        /// <inheritdoc/>
+        public override string ToString()
+        {
+            return $"(AudioTransceiver)\"{Name}\"";
+        }
+    }
+}

libs/Microsoft.MixedReality.WebRTC/DataChannel.cs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Runtime.InteropServices;
+using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 using Microsoft.MixedReality.WebRTC.Tracing;
 
@@ -97,13 +97,13 @@ public enum ChannelState
         public ChannelState State { get; private set; }
 
         /// <summary>
-        /// Event fired when the data channel state changes, as reported by <see cref="State"/>.
+        /// Event invoked when the data channel state changes, as reported by <see cref="State"/>.
         /// </summary>
         /// <seealso cref="State"/>
         public event Action StateChanged;
 
         /// <summary>
-        /// Event fired when the data channel buffering changes. Monitor this to ensure calls to
+        /// Event invoked when the data channel buffering changes. Monitor this to ensure calls to
         /// <see cref="SendMessage(byte[])"/> do not fail. Internally the data channel contains
         /// a buffer of messages to send that could not be sent immediately, for example due to
         /// congestion control. Once this buffer is full, any further call to <see cref="SendMessage(byte[])"/>
@@ -122,17 +122,21 @@ public enum ChannelState
         /// GC handle keeping the internal delegates alive while they are registered
         /// as callbacks with the native code.
         /// </summary>
-        private GCHandle _handle;
+        private IntPtr _argsRef;
 
         /// <summary>
-        /// Handle to the native object this wrapper is associated with.
+        /// Handle to the native DataChannel object.
         /// </summary>
-        internal IntPtr _interopHandle = IntPtr.Zero;
+        /// <remarks>
+        /// In native land this is a <code>mrsDataChannelHandle</code>.
+        /// </remarks>
+        internal IntPtr _nativeHandle = IntPtr.Zero;
 
-        internal DataChannel(PeerConnection peer, GCHandle handle,
-            int id, string label, bool ordered, bool reliable)
+        internal DataChannel(IntPtr nativeHandle, PeerConnection peer, IntPtr argsRef, int id, string label, bool ordered, bool reliable)
         {
-            _handle = handle;
+            Debug.Assert(nativeHandle != IntPtr.Zero);
+            _nativeHandle = nativeHandle;
+            _argsRef = argsRef;
             PeerConnection = peer;
             ID = id;
             Label = label;
@@ -156,10 +160,11 @@ internal DataChannel(PeerConnection peer, GCHandle handle,
         public void Dispose()
         {
             State = ChannelState.Closing;
-            PeerConnection.RemoveDataChannel(_interopHandle);
-            _interopHandle = IntPtr.Zero;
+            PeerConnection.RemoveDataChannel(_nativeHandle);
+            _nativeHandle = IntPtr.Zero;
             State = ChannelState.Closed;
-            _handle.Free();
+            Utils.ReleaseWrapperRef(_argsRef);
+            _argsRef = IntPtr.Zero;
             GC.SuppressFinalize(this);
         }
 
@@ -169,15 +174,21 @@ public void Dispose()
         /// The internal buffering is monitored via the <see cref="BufferingChanged"/> event.
         /// </summary>
         /// <param name="message">The message to send to the remote peer.</param>
-        /// <exception xref="InvalidOperationException">The native data channel is not initialized.</exception>
-        /// <exception xref="Exception">The internal buffer is full.</exception>
+        /// <exception xref="System.InvalidOperationException">The native data channel is not initialized.</exception>
+        /// <exception xref="System.Exception">The internal buffer is full.</exception>
+        /// <exception cref="DataChannelNotOpenException">The data channel is not open yet.</exception>
         /// <seealso cref="PeerConnection.InitializeAsync"/>
         /// <seealso cref="PeerConnection.Initialized"/>
         /// <seealso cref="BufferingChanged"/>
         public void SendMessage(byte[] message)
         {
             MainEventSource.Log.DataChannelSendMessage(ID, message.Length);
-            uint res = DataChannelInterop.DataChannel_SendMessage(_interopHandle, message, (ulong)message.LongLength);
+            // Check channel state before doing a P/Invoke call which would anyway fail
+            if (State != ChannelState.Open)
+            {
+                throw new DataChannelNotOpenException();
+            }
+            uint res = DataChannelInterop.DataChannel_SendMessage(_nativeHandle, message, (ulong)message.LongLength);
             Utils.ThrowOnErrorCode(res);
         }
 

libs/Microsoft.MixedReality.WebRTC/Exceptions.cs

@@ -58,4 +58,32 @@ public InvalidInteropNativeHandleException(string message, Exception inner)
         {
         }
     }
+
+    /// <summary>
+    /// Exception thrown when trying to use a data channel that is not open.
+    ///
+    /// The user should listen to the <see cref="DataChannel.StateChanged"/> event until the
+    /// <see cref="DataChannel.State"/> property is <see cref="DataChannel.ChannelState.Open"/>
+    /// before trying to send some message with <see cref="DataChannel.SendMessage(byte[])"/>.
+    /// </summary>
+    public class DataChannelNotOpenException : Exception
+    {
+        /// <inheritdoc/>
+        public DataChannelNotOpenException()
+            : base("Data channel is not open yet, cannot send message. Wait for the StateChanged event until State is Open.")
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message)
+            : base(message)
+        {
+        }
+
+        /// <inheritdoc/>
+        public DataChannelNotOpenException(string message, Exception inner)
+            : base(message, inner)
+        {
+        }
+    }
 }

libs/Microsoft.MixedReality.WebRTC/ExternalVideoTrackSource.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using Microsoft.MixedReality.WebRTC.Interop;
 
@@ -81,10 +82,14 @@ public void CompleteRequest(in Argb32VideoFrame frame)
     public class ExternalVideoTrackSource : IDisposable
     {
         /// <summary>
-        /// Once the external video track source is attached to some video track(s), this returns the peer connection
-        /// the video track(s) are part of. Otherwise this returns <c>null</c>.
+        /// A name for the external video track source, used for logging and debugging.
         /// </summary>
-        public PeerConnection PeerConnection { get; private set; } = null;
+        public string Name { get; set; } = string.Empty;
+
+        /// <summary>
+        /// List of local video tracks this source is providing raw video frames to.
+        /// </summary>
+        public List<LocalVideoTrack> Tracks { get; } = new List<LocalVideoTrack>();
 
         /// <summary>
         /// Handle to the native ExternalVideoTrackSource object.
@@ -164,10 +169,6 @@ public void Dispose()
                 return;
             }
 
-            // Remove the tracks associated with this source from the peer connection, if any
-            PeerConnection?.RemoveLocalVideoTracksFromSource(this);
-            Debug.Assert(PeerConnection == null); // see OnTracksRemovedFromSource
-
             // Unregister and release the track callbacks
             ExternalVideoTrackSourceInterop.ExternalVideoTrackSource_Shutdown(_nativeHandle);
             Utils.ReleaseWrapperRef(_frameRequestCallbackArgsHandle);
@@ -177,22 +178,43 @@ public void Dispose()
             _nativeHandle.Dispose();
         }
 
-        internal void OnTracksAddedToSource(PeerConnection newConnection)
+        internal void OnTrackAddedToSource(LocalVideoTrack track)
+        {
+            Debug.Assert(!_nativeHandle.IsClosed);
+            Debug.Assert(!Tracks.Contains(track));
+            Tracks.Add(track);
+        }
+
+        internal void OnTrackRemovedFromSource(LocalVideoTrack track)
         {
-            Debug.Assert(PeerConnection == null);
             Debug.Assert(!_nativeHandle.IsClosed);
-            PeerConnection = newConnection;
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = newConnection;
+            bool removed = Tracks.Remove(track);
+            Debug.Assert(removed);
         }
 
-        internal void OnTracksRemovedFromSource(PeerConnection previousConnection)
+        internal void OnTracksRemovedFromSource(List<LocalVideoTrack> tracks)
         {
-            Debug.Assert(PeerConnection == previousConnection);
             Debug.Assert(!_nativeHandle.IsClosed);
-            var args = Utils.ToWrapper<ExternalVideoTrackSourceInterop.VideoFrameRequestCallbackArgs>(_frameRequestCallbackArgsHandle);
-            args.Peer = null;
-            PeerConnection = null;
+            var remainingTracks = new List<LocalVideoTrack>();
+            foreach (var track in tracks)
+            {
+                if (track.Source == this)
+                {
+                    bool removed = Tracks.Remove(track);
+                    Debug.Assert(removed);
+                }
+                else
+                {
+                    remainingTracks.Add(track);
+                }
+            }
+            tracks = remainingTracks;
+        }
+
+        /// <inheritdoc/>
+        public override string ToString()
+        {
+            return $"(ExternalVideoTrackSource)\"{Name}\"";
         }
     }
 }

libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackInterop.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.MixedReality.WebRTC.Interop
+{
+    internal class AudioTrackInterop
+    {
+        #region Native callbacks
+
+        // The callbacks below ideally would use 'in', but that generates an error with .NET Native:
+        // "error : ILT0021: Could not resolve method 'EETypeRva:0x--------'".
+        // So instead use 'ref' to ensure the signature is compatible with the C++ const& signature.
+
+        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Ansi)]
+        public delegate void AudioFrameUnmanagedCallback(IntPtr userData, ref AudioFrame frame);
+
+        #endregion
+    }
+}