Add multi-track support

Microsoft.MixedReality.WebRTC.sln

@@ -34,6 +34,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestNetCoreConsole", "examp
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestReceiveAV", "examples\TestReceiveAV\TestReceiveAV.csproj", "{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}"
+	ProjectSection(ProjectDependencies) = postProject
+		{BD9DCE7E-7159-4DAA-AAEB-71A8261C02FF} = {BD9DCE7E-7159-4DAA-AAEB-71A8261C02FF}
+		{928899BC-F131-4343-A1AB-72F3A5787E41} = {928899BC-F131-4343-A1AB-72F3A5787E41}
+	EndProjectSection
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -98,13 +102,11 @@ Global
 		{928899BC-F131-4343-A1AB-72F3A5787E41}.Release|x86.ActiveCfg = Release|Win32
 		{928899BC-F131-4343-A1AB-72F3A5787E41}.Release|x86.Build.0 = Release|Win32
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|ARM.ActiveCfg = Debug|Any CPU
-		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|ARM.Build.0 = Debug|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|x64.Build.0 = Debug|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Debug|x86.Build.0 = Debug|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Release|ARM.ActiveCfg = Release|Any CPU
-		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Release|ARM.Build.0 = Release|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Release|x64.ActiveCfg = Release|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Release|x64.Build.0 = Release|Any CPU
 		{70AB2CE0-D35D-4911-AC83-545A611EA930}.Release|x86.ActiveCfg = Release|Any CPU
@@ -120,25 +122,21 @@ Global
 		{6D020425-2E3E-4BA7-BC46-00C8D29081C0}.Release|x86.ActiveCfg = Release|Win32
 		{6D020425-2E3E-4BA7-BC46-00C8D29081C0}.Release|x86.Build.0 = Release|Win32
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|ARM.ActiveCfg = Debug|Any CPU
-		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|ARM.Build.0 = Debug|Any CPU
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|x64.ActiveCfg = Debug|x64
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|x64.Build.0 = Debug|x64
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|x86.ActiveCfg = Debug|x86
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Debug|x86.Build.0 = Debug|x86
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|ARM.ActiveCfg = Release|Any CPU
-		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|ARM.Build.0 = Release|Any CPU
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|x64.ActiveCfg = Release|x64
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|x64.Build.0 = Release|x64
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|x86.ActiveCfg = Release|x86
 		{C17D2554-9409-4CC7-8337-E3FBE3CAE415}.Release|x86.Build.0 = Release|x86
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|ARM.ActiveCfg = Debug|Any CPU
-		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|ARM.Build.0 = Debug|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|x64.ActiveCfg = Debug|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|x64.Build.0 = Debug|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|x86.ActiveCfg = Debug|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Debug|x86.Build.0 = Debug|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Release|ARM.ActiveCfg = Release|Any CPU
-		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Release|ARM.Build.0 = Release|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Release|x64.ActiveCfg = Release|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Release|x64.Build.0 = Release|Any CPU
 		{209D1A4C-96F1-4F5E-9987-8C64E7998CC3}.Release|x86.ActiveCfg = Release|Any CPU

docs/index.md

@@ -11,6 +11,7 @@ title: Index
 - Getting started
   - [Download](manual/download.md)
   - [Installation](manual/installation.md)
+  - [Migration Guide](manual/migration-guide.md)
   - [Building from sources](manual/building.md)
   - [C# tutorial (Desktop)](manual/cs/helloworld-cs-core3.md)
   - [C# tutorial (UWP)](manual/cs/helloworld-cs-uwp.md)
@@ -50,5 +51,5 @@ title: Index
       - The PeerConnection component builds on the same-named library class to expose a remote peer connection.
    2. [Signaler](xref:Microsoft.MixedReality.WebRTC.Unity.Signaler)
       - The abstract Signaler component is the base class for signaling implementations.
-   3. [LocalVideoSource](xref:Microsoft.MixedReality.WebRTC.Unity.LocalVideoSource)
-      - The LocalVideoSource component provides access to the local webcam for local rendering and remote streaming.
+   3. [WebcamSource](xref:Microsoft.MixedReality.WebRTC.Unity.WebcamSource)
+      - The WebcamSource component provides access to the local webcam for local rendering and remote streaming.

docs/manual/cs/cs-peerconnection.md

@@ -12,12 +12,14 @@ A [`PeerConnection`](xref:Microsoft.MixedReality.WebRTC.PeerConnection) instance
 [`InitializeAsync()`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.InitializeAsync*) takes a [`PeerConnectionConfiguration`](xref:Microsoft.MixedReality.WebRTC.PeerConnectionConfiguration) and a [`CancellationToken`](xref:System.Threading.CancellationToken). The former allows configuring the peer connection about to be established, while the later allows cancelling that task while it is being processed.
 
 [`PeerConnectionConfiguration`](xref:Microsoft.MixedReality.WebRTC.PeerConnectionConfiguration) contains several fields, but the most important are:
+
 - [`IceServers`](xref:Microsoft.MixedReality.WebRTC.PeerConnectionConfiguration.IceServers) contains an optional collection of STUN and/or TURN servers used by the Interactive Connectivity Establishment (ICE) to establish a connection with the remote peer through routing devices (NATs). Without these, only direct connections to the remote peer can be established, which can be enough if the application knows in advance the network topology surrounding the two peers, but is generally recommended otherwise to increase the chances of establishing a connection over the Internet.
 - [`SdpSemantic`](xref:Microsoft.MixedReality.WebRTC.PeerConnectionConfiguration.SdpSemantic) describes the semantic used by the Session Description Protocol (SDP) while trying to establish a connection. This is a compatibility feature, which allows connecting with older peers supporting only the deprecated _Plan B_ semantic. New code should always use the default _Unified Plan_, which is the only one accepted by the WebRTC 1.0 standard.
 
 The other fields are for advanced use, and can be left with their default values.
 
 Once the connection is initialized, two categories of actions can be performed, in any order:
+
 - Attempt to establish a connection with the remote peer.
 - Add some media tracks and/or data channels.
 
@@ -26,39 +28,44 @@ Media (audio and/or video) tracks and data channels added before the connection
 ## Events
 
 A [`PeerConnection`](xref:Microsoft.MixedReality.WebRTC.PeerConnection) exposes several categories of events:
+
 - **Signaling events** are related to establishing a connection to the remote peer.
-- **Track and channel events** provide informational notifications about adding and removing media tracks and data channels.
-- **Frame events** are high frequency per-frame callbacks invoked when and audio or video frame is available for local rendering.
+- **Media and data channel events** provide informational notifications about adding and removing transceivers, media tracks, and data channels.
 
-In general it is recommended to subscribe to these events before starting to establish a connection with the remote peer.
+In general it is strongly recommended to subscribe to these events before starting to establish a connection to the remote peer.
 
 ### Signaling events
 
 | Event | Status* | Description |
 |---|---|---|
-| [`Connected`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.Connected) | Recommended |Invoked when the connection with the remote peer is established. This indicates that tracks and channel are ready for use, although ICE can continue its discovery in the background. |
-| [`LocalSdpReadytoSend`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.LocalSdpReadytoSend) | Mandatory | Invoked when the local peer finished crafting an SDP message, to request the user to send that message to the remote peer via its chosen signaling solution. |
-| <span style="white-space: pre; word-break: break-word">[`IceCandidateReadytoSend`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceCandidateReadytoSend)</span> | Mandatory | Invoked when the local peer finished crafting an ICE candidate message, to request the user to send that message to the remote peer via its chosen signaling solution. |
-| [`IceStateChanged`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceCandidateReadytoSend) | Optional | Invoked when the state of the local ICE connection changed. |
-| [`RenegotiationNeeded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceCandidateReadytoSend) | Recommended | Invoked when the peer connection detected that the current session is obsolete, and needs to be renegotiated. This is generally the result of some media tracks or data channels being added or removed, and **must** be handled to make those changes available to the remote peer. However, it is perfectly acceptable to ignore **some** of those events, if several changes are expected in a short period of time, to avoid triggering multiple unnecessary renegotiations. However a renegotiation eventually needs to happen for the newly added tracks and channel to become open. |
+| [`LocalSdpReadytoSend`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.LocalSdpReadytoSend) | Mandatory | Raised when the local peer finished crafting an SDP message, to request the user to send that message to the remote peer via its chosen signaling solution. Failing to handle this event prevents the peer-to-peer connection from establishing. |
+| <span style="white-space: pre; word-break: break-word">[`IceCandidateReadytoSend`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceCandidateReadytoSend)</span> | Mandatory | Raised when the local peer finished crafting an ICE candidate message, to request the user to send that message to the remote peer via its chosen signaling solution. Failing to handle this event generally prevents the peer-to-peer connection from establishing. |
+| [`Connected`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.Connected) | Recommended |Raised when the connection with the remote peer is established. This indicates that tracks and channel are ready for use, although ICE can continue its discovery in the background. This does **not** indicate that the session negotiation is completed yet; this event is raised on the answering peer as soon as it creates an answer and applies it, before it is sent to the remote peer, while it is only raised later on the offering peer when that answer is actually received and applied. |
+| [`RenegotiationNeeded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceCandidateReadytoSend) | Recommended | Raised when the peer connection detected that the current session is obsolete, and needs to be renegotiated. This is generally the result of some media tracks or data channels being added or removed, and **must** be handled to make those changes available to the remote peer. However, it is perfectly acceptable to ignore **some** of those events, if several changes are expected in a short period of time, to avoid triggering multiple unnecessary renegotiations. However a renegotiation eventually needs to happen for the newly added tracks and channel to become open. |
+| [`IceStateChanged`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceStateChanged) | Optional | Raised when the state of the local ICE connection changed. |
+| [`IceGatheringStateChanged`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.IceGatheringStateChanged) | Optional | Raised when the state of the ICE candidate gathering process changed. |
 
 *_Status_ indicates the recommended subscription status for a working peer connection:
+
 - **Mandatory** means those events must be handled, otherwise the connection cannot be established.
 - **Recommended** means those events are typically handled, although not mandatory.
 - **Optional** means those events are informational only, and it is entirely optional to subscribe to them.
 
-### Track and channel events
+### Media and data channel events
 
 | Event | Status* | Description |
 |---|---|---|
-| [`TrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.TrackAdded) | Optional | Invoked when a media track (audio or video) was added on the remote peer, and received on the local peer after a renegotiation. This only concerns remotely-created tracks. Tracks whose creation was initied locally with _e.g._ [`AddLocalVideoTrackAsync()`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AddLocalVideoTrackAsync(Microsoft.MixedReality.WebRTC.PeerConnection.LocalVideoTrackSettings)) do not generate a [`TrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.TrackAdded) event. |
-| [`TrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.TrackRemoved) | Optional | Invoked when a media track (audio or video) was removed on the remote peer, and a remove message was received on the local peer after a renegotiation. This only concerns remotely-deleted tracks. Tracks removed locally with _e.g._ [`RemoveLocalVideoTrack(LocalVideoTrack)`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.RemoveLocalVideoTrack(Microsoft.MixedReality.WebRTC.LocalVideoTrack)) do not generate a [`TrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.TrackRemoved) event. |
+| [`TransceiverAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.TransceiverAdded) | Optional | Raised when a transceiver was added to the peer connection, either manually by calling [`AddTransceiver()`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AddTransceiver(Microsoft.MixedReality.WebRTC.MediaKind,Microsoft.MixedReality.WebRTC.TransceiverInitSettings)), or automatically by the implementation when a remote session description is applied with [`SetRemoteDescriptionAsync()`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.SetRemoteDescriptionAsync(System.String,System.String)) and the remote peer added a new transceiver. |
+| [`AudioTrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AudioTrackAdded) | Optional | Raised when an audio track was added on the remote peer, and received on the local peer after a session negotiation. This only concerns remotely-created tracks. Tracks whose creation was initied locally with _e.g._ [`LocalAudioTrack.CreateFromDeviceAsync()`](xref:Microsoft.MixedReality.WebRTC.LocalAudioTrack.CreateFromDeviceAsync(Microsoft.MixedReality.WebRTC.LocalAudioTrackSettings)) do not raise the [`AudioTrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AudioTrackAdded) event. |
+| [`AudioTrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AudioTrackRemoved) | Optional | Invoked when an audio track was removed on the remote peer, and a remove message was received on the local peer after a session negotiation. This only concerns remote tracks. Local tracks do not raise the [`AudioTrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.AudioTrackRemoved) event. |
+| [`VideoTrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.VideoTrackAdded) | Optional | Invoked when a video track was added on the remote peer, and received on the local peer after a session negotiation. This only concerns remotely-created tracks. Tracks whose creation was initied locally with _e.g._ [`LocalVideoTrack.CreateFromDeviceAsync()`](xref:Microsoft.MixedReality.WebRTC.LocalVideoTrack.CreateFromDeviceAsync(Microsoft.MixedReality.WebRTC.LocalVideoTrackSettings)) do not raise the [`VideoTrackAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.VideoTrackAdded) event. |
+| [`VideoTrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.VideoTrackRemoved) | Optional | Invoked when a video track was removed on the remote peer, and a remove message was received on the local peer after a session negotiation. This only concerns remote tracks. Local tracks do not raise the [`VideoTrackRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.VideoTrackRemoved) event. |
 | [`DataChannelAdded`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.DataChannelAdded) | Optional | Invoked when a data channel was added to the local peer connection. This is always fired, irrelevant of how the data channel was initially created (in-band or out-of-band). |
 | <span style="white-space: pre; word-break: break-word">[`DataChannelRemoved`](xref:Microsoft.MixedReality.WebRTC.PeerConnection.DataChannelRemoved)</span> | Optional | Invoked when a data channel was removed from the local peer connection. This is always fired, irrelevant of how the data channel was initially created (in-band or out-of-band). |
 
 *_Status_ - See above.
 
-### Frame events
+<!-- ### Frame events
 
 | Event | Status* | Description |
 |---|---|---|
@@ -70,4 +77,4 @@ In general it is recommended to subscribe to these events before starting to est
 *_Status_ - See above.
 
 > [!NOTE]
-> It is generally recommended to use the I420 callbacks instead of the ARGB ones. Even in situations where the processing (_e.g._ local rendering) requires ARGB frames, I420 frames are smaller thanks to the chroma downsampling, so faster to upload to GPU. And GPU-based I420-to-ARGB conversion via a custom pixel shader (fragment shader) is more efficient than the CPU conversion provided by the ARGB callbacks, even with the use of SIMD. See [`YUVFeedShaderUnlit.shader`](https://github.com/microsoft/MixedReality-WebRTC/blob/master/libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity/Shaders/YUVFeedShaderUnlit.shader) in the Unity integration for an example of such conversion shader.
+> It is generally recommended to use the I420 callbacks instead of the ARGB ones. Even in situations where the processing (_e.g._ local rendering) requires ARGB frames, I420 frames are smaller thanks to the chroma downsampling, so faster to upload to GPU. And GPU-based I420-to-ARGB conversion via a custom pixel shader (fragment shader) is more efficient than the CPU conversion provided by the ARGB callbacks, even with the use of SIMD. See [`YUVFeedShaderUnlit.shader`](https://github.com/microsoft/MixedReality-WebRTC/blob/master/libs/Microsoft.MixedReality.WebRTC.Unity/Assets/Microsoft.MixedReality.WebRTC.Unity/Shaders/YUVFeedShaderUnlit.shader) in the Unity integration for an example of such conversion shader. -->