Skip to content
This repository has been archived by the owner on Mar 22, 2022. It is now read-only.

Commit

Permalink
Enable multiple local video tracks per connection
Browse files Browse the repository at this point in the history 
This change introduces a standalone LocalVideoTrack object encapsulating
a video track originating from the local peer and sent to the remote
peer. This is an extension of the previously existing single local video
track internally owned by the PeerConnection object, now existing as a
standalone object.

This change enables several new features and benefits:
- The PeerConnection object now holds references to the LocalVideoTrack
object, instead of owning it internally. This makes it possible to add
multiple local video tracks per peer connection.
- Interactions with the local video track, like enabling/disabling the
track, is achieved by direct manipulation of the LocalVideoTrack object
instance, without the need to know of nor keep a reference to the
PeerConnection object the track may be attached to.
- Looking forward, this extracting of the local video track, currently
backed by a local video capture device, from the peer connection enables
future changes to add new kinds of local video tracks backed by
different track sources.
  • Loading branch information
@djee-ms
djee-ms committed Nov 18, 2019
1 parent ddaef4d commit 1050e49
Show file tree
Hide file tree
Showing 27 changed files with 1,011 additions and 369 deletions.
71 changes: 44 additions & 27 deletions examples/TestAppUwp/MainPage.xaml.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -71,6 +71,7 @@ public sealed partial class MainPage : Page
private MediaPlayer localVideoPlayer = new MediaPlayer();
private bool _isLocalVideoPlaying = false;
private object _isLocalVideoPlayingLock = new object();
private LocalVideoTrack _localVideoTrack = null;

private MediaStreamSource remoteVideoSource = null;
private MediaSource remoteMediaSource = null;
Expand Down Expand Up @@ -235,7 +236,6 @@ public MainPage()
_peerConnection.RenegotiationNeeded += OnPeerRenegotiationNeeded;
_peerConnection.TrackAdded += Peer_RemoteTrackAdded;
_peerConnection.TrackRemoved += Peer_RemoteTrackRemoved;
_peerConnection.I420LocalVideoFrameReady += Peer_LocalI420FrameReady;
_peerConnection.I420RemoteVideoFrameReady += Peer_RemoteI420FrameReady;
_peerConnection.LocalAudioFrameReady += Peer_LocalAudioFrameReady;
_peerConnection.RemoteAudioFrameReady += Peer_RemoteAudioFrameReady;
Expand Down Expand Up @@ -855,7 +855,7 @@ private void OnMediaEnded(MediaPlayer sender, object args)
{
RunOnMainThread(() => {
LogMessage("Local MediaElement video playback ended.");
//StopLocalVideo();
//StopLocalMedia();
sender.Pause();
sender.Source = null;
if (sender == localVideoPlayer)
Expand All @@ -875,7 +875,7 @@ private void OnMediaEnded(MediaPlayer sender, object args)
/// audio and video tracks from the peer connection.
/// This is called on the UI thread.
/// </summary>
private void StopLocalVideo()
private async void StopLocalMedia()
{
lock (_isLocalVideoPlayingLock)
{
Expand All @@ -889,22 +889,29 @@ private void StopLocalVideo()
localMediaSource.Dispose();
localMediaSource = null;
_isLocalVideoPlaying = false;

// Avoid deadlock in audio processing stack, as this call is delegated to the WebRTC
// signaling thread (and will block the caller thread), and audio processing will
// delegate to the UI thread for UWP operations (and will block the signaling thread).
RunOnWorkerThread(() => {
_renegotiationOfferEnabled = false;
_peerConnection.RemoveLocalAudioTrack();
_peerConnection.RemoveLocalVideoTrack();
_renegotiationOfferEnabled = true;
if (_peerConnection.IsConnected)
{
_peerConnection.CreateOffer();
}
});
_localVideoTrack.I420VideoFrameReady -= LocalVideoTrack_I420FrameReady;
}
}

// Avoid deadlock in audio processing stack, as this call is delegated to the WebRTC
// signaling thread (and will block the caller thread), and audio processing will
// delegate to the UI thread for UWP operations (and will block the signaling thread).
await RunOnWorkerThread(() => {
lock (_isLocalVideoPlayingLock)
{
_peerConnection.RemoveLocalAudioTrack();
_peerConnection.RemoveLocalVideoTrack(_localVideoTrack); // TODO - this doesn't unregister the callbacks...
_renegotiationOfferEnabled = true;
if (_peerConnection.IsConnected)
{
_peerConnection.CreateOffer();
}
_localVideoTrack.Dispose();
_localVideoTrack = null;
}
});

startLocalMedia.IsEnabled = true;
}

/// <summary>
Expand Down Expand Up @@ -962,7 +969,7 @@ private void Peer_RemoteTrackRemoved(PeerConnection.TrackKind trackKind)
/// for local rendering before (or in parallel of) being sent to the remote peer.
/// </summary>
/// <param name="frame">The newly captured video frame.</param>
private void Peer_LocalI420FrameReady(I420AVideoFrame frame)
private void LocalVideoTrack_I420FrameReady(I420AVideoFrame frame)
{
localVideoBridge.HandleIncomingVideoFrame(frame);
}
Expand Down Expand Up @@ -1064,14 +1071,14 @@ private void UpdateRemoteAudioStats(uint channelCount, uint sampleRate)

private void MuteLocalVideoClicked(object sender, RoutedEventArgs e)
{
if (_peerConnection.IsLocalVideoTrackEnabled())
if (_localVideoTrack.Enabled)
{
_peerConnection.SetLocalVideoTrackEnabled(false);
_localVideoTrack.Enabled = false;
muteLocalVideoStroke.Visibility = Visibility.Visible;
}
else
{
_peerConnection.SetLocalVideoTrackEnabled(true);
_localVideoTrack.Enabled = true;
muteLocalVideoStroke.Visibility = Visibility.Collapsed;
}
}
Expand All @@ -1097,12 +1104,15 @@ private void MuteLocalAudioClicked(object sender, RoutedEventArgs e)
/// <param name="e">Event arguments.</param>
private async void StartLocalMediaClicked(object sender, RoutedEventArgs e)
{
// The button will be re-enabled once the current action is finished
startLocalMedia.IsEnabled = false;

// Toggle between start and stop local audio/video feeds
//< TODO dssStatsTimer.IsEnabled used for toggle, but dssStatsTimer should be
// used also for remote statistics display (so even when no local video active)
if (dssStatsTimer.IsEnabled)
if (dssStatsTimer.IsEnabled && (_localVideoTrack != null))
{
StopLocalVideo();
StopLocalMedia();
dssStatsTimer.Stop();
localLoadText.Text = "Load: -";
localPresentText.Text = "Present: -";
Expand All @@ -1117,8 +1127,11 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e)
startLocalMediaIcon.Symbol = Symbol.Play;
muteLocalAudioStroke.Visibility = Visibility.Collapsed;
muteLocalVideoStroke.Visibility = Visibility.Collapsed;
return;
}
else

// Only start video if previous track was completely shutdown
if (_localVideoTrack == null)
{
LogMessage("Opening local A/V stream...");

Expand Down Expand Up @@ -1192,7 +1205,8 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e)
framerate = framerate,
enableMrc = false // TestAppUWP is a shared app, MRC will not get permission anyway
};
await _peerConnection.AddLocalVideoTrackAsync(trackConfig);
_localVideoTrack = await _peerConnection.AddLocalVideoTrackAsync("local_video", trackConfig);
_localVideoTrack.I420VideoFrameReady += LocalVideoTrack_I420FrameReady;
}
catch (Exception ex)
{
Expand Down Expand Up @@ -1222,6 +1236,9 @@ private async void StartLocalMediaClicked(object sender, RoutedEventArgs e)
{
_peerConnection.CreateOffer();
}

// Enable stopping the local audio and video
startLocalMedia.IsEnabled = true;
}
}

Expand Down
64 changes: 64 additions & 0 deletions libs/Microsoft.MixedReality.WebRTC.Native/include/local_video_track.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

#pragma once

#include "api/mediastreaminterface.h"
#include "api/peerconnectioninterface.h"
#include "api/rtpsenderinterface.h"

#include "callback.h"
#include "interop/interop_api.h"
#include "str.h"
#include "video_frame_observer.h"

namespace Microsoft::MixedReality::WebRTC {

class PeerConnection;

class LocalVideoTrack : public VideoFrameObserver,
public rtc::RefCountInterface {
public:
LocalVideoTrack(PeerConnection& owner,
rtc::scoped_refptr<webrtc::VideoTrackInterface> track,
rtc::scoped_refptr<webrtc::RtpSenderInterface> sender,
mrsLocalVideoTrackInteropHandle interop_handle) noexcept;
MRS_API ~LocalVideoTrack() override;

MRS_API [[nodiscard]] bool IsEnabled() const noexcept;
MRS_API void SetEnabled(bool enabled) const noexcept;

//
// Advanced use
//

[[nodiscard]] webrtc::VideoTrackInterface* impl() const {
return track_.get();
}

[[nodiscard]] webrtc::RtpSenderInterface* sender() const {
return sender_.get();
}

[[nodiscard]] mrsLocalVideoTrackInteropHandle GetInteropHandle() const
noexcept {
return interop_handle_;
}

void RemoveFromPeerConnection(webrtc::PeerConnectionInterface& peer);

private:
/// PeerConnection object owning this track.
PeerConnection* owner_{};

/// Underlying core implementation.
rtc::scoped_refptr<webrtc::VideoTrackInterface> track_;

/// RTP sender this track is associated with.
rtc::scoped_refptr<webrtc::RtpSenderInterface> sender_;

/// Optional interop handle, if associated with an interop wrapper.
mrsLocalVideoTrackInteropHandle interop_handle_{};
};

} // namespace Microsoft::MixedReality::WebRTC
60 changes: 13 additions & 47 deletions libs/Microsoft.MixedReality.WebRTC.Native/include/peer_connection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
namespace Microsoft::MixedReality::WebRTC {

class PeerConnection;
class LocalVideoTrack;
class DataChannel;

/// The PeerConnection class is the entry point to most of WebRTC.
Expand Down Expand Up @@ -184,24 +185,6 @@ class PeerConnection : public webrtc::PeerConnectionObserver,
// Video
//

/// Register a custom callback invoked when a local video frame is ready to be
/// displayed.
void RegisterLocalVideoFrameCallback(
I420FrameReadyCallback callback) noexcept {
if (local_video_observer_) {
local_video_observer_->SetCallback(std::move(callback));
}
}

/// Register a custom callback invoked when a local video frame is ready to be
/// displayed.
void RegisterLocalVideoFrameCallback(
ARGBFrameReadyCallback callback) noexcept {
if (local_video_observer_) {
local_video_observer_->SetCallback(std::move(callback));
}
}

/// Register a custom callback invoked when a remote video frame has been
/// received and decompressed, and is ready to be displayed locally.
void RegisterRemoteVideoFrameCallback(
Expand All @@ -220,37 +203,14 @@ class PeerConnection : public webrtc::PeerConnectionObserver,
}
}

/// Add to the peer connection a video track backed by a local video capture
/// device. If no RTP sender/transceiver exist, create a new one for that
/// track.
///
/// Note: currently a single local video track is supported per peer
/// connection.
bool AddLocalVideoTrack(
/// Add a video track to the peer connection. If no RTP sender/transceiver
/// exist, create a new one for that track.
webrtc::RTCErrorOr<rtc::scoped_refptr<LocalVideoTrack>> AddLocalVideoTrack(
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track) noexcept;

/// Remove the existing local video track from the peer connection.
/// Remove a local video track from the peer connection.
/// The underlying RTP sender/transceiver are kept alive but inactive.
///
/// Note: currently a single local video track is supported per peer
/// connection.
void RemoveLocalVideoTrack() noexcept;

/// Enable or disable the local video track. Disabled video tracks are still
/// active but output black frames, and do not consume network bandwidth.
/// Additionally, enabling/disabling the local video track does not require an
/// SDP exchange. Therefore this is a cheaper alternative to removing and
/// re-adding the track.
///
/// Note: currently a single local video track is supported per peer
/// connection.
void SetLocalVideoTrackEnabled(bool enabled = true) noexcept;

/// Check if the local video frame is enabled.
///
/// Note: currently a single local video track is supported per peer
/// connection.
bool IsLocalVideoTrackEnabled() const noexcept;
webrtc::RTCError RemoveLocalVideoTrack(LocalVideoTrack& video_track) noexcept;

//
// Audio
Expand Down Expand Up @@ -522,6 +482,13 @@ class PeerConnection : public webrtc::PeerConnectionObserver,
rtc::scoped_refptr<webrtc::RtpSenderInterface> local_audio_sender_;
std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>> remote_streams_;

/// Collection of all local video tracks associated with this peer connection.
std::vector<rtc::scoped_refptr<LocalVideoTrack>> local_video_tracks_
RTC_GUARDED_BY(tracks_mutex_);

/// Mutex for all collections of all tracks.
rtc::CriticalSection tracks_mutex_;

/// Collection of all data channels associated with this peer connection.
std::vector<std::shared_ptr<DataChannel>> data_channels_
RTC_GUARDED_BY(data_channel_mutex_);
Expand All @@ -543,7 +510,6 @@ class PeerConnection : public webrtc::PeerConnectionObserver,
//< TODO - Clarify lifetime of those, for now same as this PeerConnection
std::unique_ptr<AudioFrameObserver> local_audio_observer_;
std::unique_ptr<AudioFrameObserver> remote_audio_observer_;
std::unique_ptr<VideoFrameObserver> local_video_observer_;
std::unique_ptr<VideoFrameObserver> remote_video_observer_;

/// Flag to indicate if SCTP was negotiated during the initial SDP handshake
Expand Down
Loading

0 comments on commit 1050e49

Please sign in to comment.