From 07233bac5fbb156f2775eb4d5c7405adc46acf97 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 18 May 2020 17:44:44 +0100 Subject: [PATCH 1/7] Add split audio track source Split the audio track source from the audio track, to allow sharing across peer connections. This change splits the former local audio track into 2 separate components: - The audio track source, which generates the raw audio frames, generally from a local audio capture device (microphone) but not necessarily. At this time only device-based sources are supported though. The audio track source is not associated with any peer connection in particular, and can be shared amongst any number of local audio tracks, from the same peer connection or not. - The local audio track, which is associated with a particular peer connection, and which is a slim bridge between an audio track source and the audio transceiver of the peer connection. --- examples/TestAppUwp/MediaPlayerPage.xaml.cs | 2 + .../ViewModel/AudioCaptureViewModel.cs | 9 +- .../ViewModel/MediaPlayerViewModel.cs | 5 +- examples/TestNetCoreConsole/Program.cs | 13 +- .../AudioTrackSource.cs | 176 ++++++ .../Interop/AudioTrackSourceInterop.cs | 107 ++++ .../Interop/LocalAudioTrackInterop.cs | 15 +- .../Interop/PeerConnectionInterop.cs | 17 - .../LocalAudioTrack.cs | 70 ++- .../include/audio_track_source_interop.h | 76 +++ libs/mrwebrtc/include/interop_api.h | 3 + .../include/local_audio_track_interop.h | 19 +- .../interop/audio_track_source_interop.cpp | 130 +++++ libs/mrwebrtc/src/interop/global_factory.cpp | 2 + libs/mrwebrtc/src/interop/interop_api.cpp | 45 +- .../src/interop/local_audio_track_interop.cpp | 49 +- .../mrwebrtc/src/media/audio_track_source.cpp | 74 +++ libs/mrwebrtc/src/media/audio_track_source.h | 116 ++++ libs/mrwebrtc/src/tracked_object.h | 1 + .../test/audio_track_source_tests.cpp | 105 ++++ libs/mrwebrtc/test/audio_track_tests.cpp | 32 +- .../AudioTrackSourceTests.cs | 47 ++ .../LocalAudioTrackTests.cs | 38 ++ tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj | 500 +++++++++--------- .../mrwebrtc/uwp/mrwebrtc-uwp.vcxproj.filters | 462 ++++++++-------- .../mrwebrtc/win32/mrwebrtc-win32.vcxproj | 430 +++++++-------- .../win32/mrwebrtc-win32.vcxproj.filters | 462 ++++++++-------- .../win32/tests/mrwebrtc-win32-tests.vcxproj | 1 + 28 files changed, 1988 insertions(+), 1018 deletions(-) create mode 100644 libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs create mode 100644 libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs create mode 100644 libs/mrwebrtc/include/audio_track_source_interop.h create mode 100644 libs/mrwebrtc/src/interop/audio_track_source_interop.cpp create mode 100644 libs/mrwebrtc/src/media/audio_track_source.cpp create mode 100644 libs/mrwebrtc/src/media/audio_track_source.h create mode 100644 libs/mrwebrtc/test/audio_track_source_tests.cpp create mode 100644 tests/Microsoft.MixedReality.WebRTC.Tests/AudioTrackSourceTests.cs create mode 100644 tests/Microsoft.MixedReality.WebRTC.Tests/LocalAudioTrackTests.cs diff --git a/examples/TestAppUwp/MediaPlayerPage.xaml.cs b/examples/TestAppUwp/MediaPlayerPage.xaml.cs index ba51acadd..d402f84ea 100644 --- a/examples/TestAppUwp/MediaPlayerPage.xaml.cs +++ b/examples/TestAppUwp/MediaPlayerPage.xaml.cs @@ -52,6 +52,8 @@ public string DisplayName /// public class AudioTrackViewModel { + // FIXME - this leaks 'source', never disposed (and is the track itself disposed??) + public AudioTrackSource Source; public IAudioTrack Track; public MediaTrack TrackImpl; public bool IsRemote; diff --git a/examples/TestAppUwp/ViewModel/AudioCaptureViewModel.cs b/examples/TestAppUwp/ViewModel/AudioCaptureViewModel.cs index 392ab09e2..7a2cc890e 100644 --- a/examples/TestAppUwp/ViewModel/AudioCaptureViewModel.cs +++ b/examples/TestAppUwp/ViewModel/AudioCaptureViewModel.cs @@ -33,14 +33,19 @@ public async Task AddAudioTrackFromDeviceAsync(string trackName) await RequestMediaAccessAsync(StreamingCaptureMode.Audio); - var settings = new LocalAudioTrackSettings + // FIXME - this leaks 'source', never disposed (and is the track itself disposed??) + var initConfig = new LocalAudioDeviceInitConfig(); + var source = await AudioTrackSource.CreateFromDeviceAsync(initConfig); + + var settings = new LocalAudioTrackInitConfig { trackName = trackName }; - var track = await LocalAudioTrack.CreateFromDeviceAsync(settings); + var track = await LocalAudioTrack.CreateFromSourceAsync(source, settings); SessionModel.Current.AudioTracks.Add(new AudioTrackViewModel { + Source = source, Track = track, TrackImpl = track, IsRemote = false, diff --git a/examples/TestAppUwp/ViewModel/MediaPlayerViewModel.cs b/examples/TestAppUwp/ViewModel/MediaPlayerViewModel.cs index 84f65014c..adeda2a88 100644 --- a/examples/TestAppUwp/ViewModel/MediaPlayerViewModel.cs +++ b/examples/TestAppUwp/ViewModel/MediaPlayerViewModel.cs @@ -117,7 +117,10 @@ public MediaPlayerViewModel() DisplayName = "Local microphone (default device)", Factory = async () => { - return await LocalAudioTrack.CreateFromDeviceAsync(); + // FIXME - this leaks 'source', never disposed (and is the track itself disposed??) + var source = await AudioTrackSource.CreateFromDeviceAsync(); + var settings = new LocalAudioTrackInitConfig(); + return await LocalAudioTrack.CreateFromSourceAsync(source, settings); } }); diff --git a/examples/TestNetCoreConsole/Program.cs b/examples/TestNetCoreConsole/Program.cs index 3e9589b62..c9ca4ec6f 100644 --- a/examples/TestNetCoreConsole/Program.cs +++ b/examples/TestNetCoreConsole/Program.cs @@ -14,6 +14,7 @@ static async Task Main(string[] args) { Transceiver audioTransceiver = null; Transceiver videoTransceiver = null; + AudioTrackSource audioTrackSource = null; LocalAudioTrack localAudioTrack = null; LocalVideoTrack localVideoTrack = null; @@ -58,7 +59,13 @@ static async Task Main(string[] args) if (needAudio) { Console.WriteLine("Opening local microphone..."); - localAudioTrack = await LocalAudioTrack.CreateFromDeviceAsync(); + audioTrackSource = await AudioTrackSource.CreateFromDeviceAsync(); + + Console.WriteLine("Create local audio track..."); + var trackSettings = new LocalAudioTrackInitConfig { trackName = "mic_track" }; + localAudioTrack = await LocalAudioTrack.CreateFromSourceAsync(audioTrackSource, trackSettings); + + Console.WriteLine("Create audio transceiver and add mic track..."); audioTransceiver = pc.AddTransceiver(MediaKind.Audio); audioTransceiver.DesiredDirection = Transceiver.Direction.SendReceive; audioTransceiver.LocalAudioTrack = localAudioTrack; @@ -118,6 +125,10 @@ static async Task Main(string[] args) localVideoTrack?.Dispose(); Console.WriteLine("Program termined."); + + localAudioTrack.Dispose(); + localVideoTrack.Dispose(); + audioTrackSource.Dispose(); } } } diff --git a/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs new file mode 100644 index 000000000..9a9aaedb9 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.MixedReality.WebRTC.Interop; + +namespace Microsoft.MixedReality.WebRTC +{ + /// + /// Configuration to initialize capture on a local audio device (microphone). + /// + public class LocalAudioDeviceInitConfig + { + /// + /// Enable automated gain control (AGC) on the audio device capture pipeline. + /// + public bool? AutoGainControl = null; + } + + /// + /// Audio source for WebRTC audio tracks. + /// + /// The audio source is not bound to any peer connection, and can therefore be shared by multiple audio + /// tracks from different peer connections. This is especially useful to share local audio capture devices + /// (microphones) amongst multiple peer connections when building a multi-peer experience with a mesh topology + /// (one connection per pair of peers). + /// + /// The user owns the audio track source, and is in charge of keeping it alive until after all tracks using it + /// are destroyed, and then dispose of it. The behavior of disposing of the track source while a track is still + /// using it is undefined. The property contains the list of tracks currently using the + /// source. + /// + /// + public class AudioTrackSource : IDisposable + { + /// + /// A name for the audio track source, used for logging and debugging. + /// + public string Name + { + get + { + // Note: the name cannot change internally, so no need to query the native layer. + // This avoids a round-trip to native and some string encoding conversion. + return _name; + } + set + { + AudioTrackSourceInterop.AudioTrackSource_SetName(_nativeHandle, value); + _name = value; + } + } + + /// + /// List of local audio tracks this source is providing raw audio frames to. + /// + public List Tracks { get; private set; } = new List(); + + /// + /// Handle to the native AudioTrackSource object. + /// + /// + /// In native land this is a Microsoft::MixedReality::WebRTC::AudioTrackSourceHandle. + /// + internal AudioTrackSourceHandle _nativeHandle { get; private set; } = new AudioTrackSourceHandle(); + + /// + /// Backing field for , and cache for the native name. + /// Since the name can only be set by the user, this cached value is always up-to-date with the + /// internal name of the native object, by design. + /// + private string _name = string.Empty; + + /// + /// Create an audio track source using a local audio capture device (microphone). + /// + /// Optional configuration to initialize the audio capture on the device. + /// The newly create audio track source. + public static Task CreateFromDeviceAsync(LocalAudioDeviceInitConfig initConfig = null) + { + return Task.Run(() => + { + // On UWP this cannot be called from the main UI thread, so always call it from + // a background worker thread. + + var config = new AudioTrackSourceInterop.LocalAudioDeviceMarshalInitConfig(initConfig); + uint ret = AudioTrackSourceInterop.AudioTrackSource_CreateFromDevice(in config, out AudioTrackSourceHandle handle); + Utils.ThrowOnErrorCode(ret); + return new AudioTrackSource(handle); + }); + } + + internal AudioTrackSource(AudioTrackSourceHandle nativeHandle) + { + _nativeHandle = nativeHandle; + } + + /// + public void Dispose() + { + if (_nativeHandle.IsClosed) + { + return; + } + + // TODO - Can we support destroying the source and leaving tracks with silence instead? + if (Tracks.Count > 0) + { + throw new InvalidOperationException($"Trying to dispose of AudioTrackSource '{Name}' while still in use by one or more audio tracks."); + } + + // Unregister from tracks + // TODO... + //AudioTrackSourceInterop.AudioTrackSource_Shutdown(_nativeHandle); + + // Destroy the native object. This may be delayed if a P/Invoke callback is underway, + // but will be handled at some point anyway, even if the managed instance is gone. + _nativeHandle.Dispose(); + } + + /// + /// Internal callback when a track starts using this source. + /// + /// The track using this source. + internal void OnTrackAddedToSource(LocalAudioTrack track) + { + Debug.Assert(!_nativeHandle.IsClosed); + Debug.Assert(!Tracks.Contains(track)); + Tracks.Add(track); + } + + /// + /// Internal callback when a track stops using this source. + /// + /// The track not using this source anymore. + internal void OnTrackRemovedFromSource(LocalAudioTrack track) + { + Debug.Assert(!_nativeHandle.IsClosed); + bool removed = Tracks.Remove(track); + Debug.Assert(removed); + } + + /// + /// Internal callback when a list of tracks stop using this source, generally + /// as a result of a peer connection owning said tracks being closed. + /// + /// The list of tracks not using this source anymore. + internal void OnTracksRemovedFromSource(List tracks) + { + Debug.Assert(!_nativeHandle.IsClosed); + var remainingTracks = new List(); + foreach (var track in tracks) + { + if (track.Source == this) + { + bool removed = Tracks.Remove(track); + Debug.Assert(removed); + } + else + { + remainingTracks.Add(track); + } + } + Tracks = remainingTracks; + } + + /// + public override string ToString() + { + return $"(AudioTrackSource)\"{Name}\""; + } + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs new file mode 100644 index 000000000..94339c306 --- /dev/null +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Microsoft.MixedReality.WebRTC.Interop +{ + /// + /// Handle to a native audio track source object. + /// + internal sealed class AudioTrackSourceHandle : SafeHandle + { + /// + /// Check if the current handle is invalid, which means it is not referencing + /// an actual native object. Note that a valid handle only means that the internal + /// handle references a native object, but does not guarantee that the native + /// object is still accessible. It is only safe to access the native object if + /// the handle is not closed, which implies it being valid. + /// + public override bool IsInvalid => (handle == IntPtr.Zero); + + /// + /// Default constructor for an invalid handle. + /// + public AudioTrackSourceHandle() : base(IntPtr.Zero, ownsHandle: true) + { + } + + /// + /// Constructor for a valid handle referencing the given native object. + /// + /// The valid internal handle to the native object. + public AudioTrackSourceHandle(IntPtr handle) : base(IntPtr.Zero, ownsHandle: true) + { + SetHandle(handle); + } + + /// + /// Release the native object while the handle is being closed. + /// + /// Return true if the native object was successfully released. + protected override bool ReleaseHandle() + { + AudioTrackSourceInterop.AudioTrackSource_RemoveRef(handle); + return true; + } + } + + internal class AudioTrackSourceInterop + { + /// + /// Marshaling struct for initializing settings when opening a local audio device. + /// + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal ref struct LocalAudioDeviceMarshalInitConfig + { + public mrsOptBool AutoGainControl; + + /// + /// Constructor for creating a local audio device initialization settings marshaling struct. + /// + /// The settings to initialize the newly created marshaling struct. + /// + public LocalAudioDeviceMarshalInitConfig(LocalAudioDeviceInitConfig settings) + { + AutoGainControl = new mrsOptBool(settings?.AutoGainControl); + } + } + + #region P/Invoke static functions + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceAddRef")] + public static unsafe extern void AudioTrackSource_AddRef(AudioTrackSourceHandle handle); + + // Note - This is used during SafeHandle.ReleaseHandle(), so cannot use AudioTrackSourceHandle + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceRemoveRef")] + public static unsafe extern void AudioTrackSource_RemoveRef(IntPtr handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceSetName")] + public static unsafe extern void AudioTrackSource_SetName(AudioTrackSourceHandle handle, string name); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceGetName")] + public static unsafe extern uint AudioTrackSource_GetName(AudioTrackSourceHandle handle, StringBuilder buffer, + ref ulong bufferCapacity); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceSetUserData")] + public static unsafe extern void AudioTrackSource_SetUserData(AudioTrackSourceHandle handle, IntPtr userData); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceGetUserData")] + public static unsafe extern IntPtr AudioTrackSource_GetUserData(AudioTrackSourceHandle handle); + + [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, + EntryPoint = "mrsAudioTrackSourceCreateFromDevice")] + public static unsafe extern uint AudioTrackSource_CreateFromDevice( + in LocalAudioDeviceMarshalInitConfig config, out AudioTrackSourceHandle sourceHandle); + + #endregion + } +} diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs index 180d03c15..2b2483016 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/LocalAudioTrackInterop.cs @@ -56,6 +56,14 @@ protected override bool ReleaseHandle() internal class LocalAudioTrackInterop { + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal ref struct TrackInitConfig + { + [MarshalAs(UnmanagedType.LPStr)] + public string TrackName; + } + + #region Native functions [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, @@ -68,9 +76,9 @@ internal class LocalAudioTrackInterop public static unsafe extern void LocalAudioTrack_RemoveRef(IntPtr handle); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, - EntryPoint = "mrsLocalAudioTrackCreateFromDevice")] - public static unsafe extern uint LocalAudioTrack_CreateFromDevice(in PeerConnectionInterop.LocalAudioTrackInteropInitConfig config, - string trackName, out LocalAudioTrackHandle trackHandle); + EntryPoint = "mrsLocalAudioTrackCreateFromSource")] + public static unsafe extern uint LocalAudioTrack_CreateFromSource(in TrackInitConfig initSettings, + AudioTrackSourceHandle sourceHandle, out LocalAudioTrackHandle trackHandle); [DllImport(Utils.dllPath, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi, EntryPoint = "mrsLocalAudioTrackRegisterFrameCallback")] @@ -87,6 +95,7 @@ public static extern void LocalAudioTrack_RegisterFrameCallback(LocalAudioTrackH #endregion + public class InteropCallbackArgs { public LocalAudioTrack Track; diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs index 0517210a4..c03f07c56 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/PeerConnectionInterop.cs @@ -350,23 +350,6 @@ internal ref struct IceCandidate public int SdpMlineIndex; } - /// - /// Helper structure to pass parameters to the native implementation when creating a local audio track. - /// - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal ref struct LocalAudioTrackInteropInitConfig - { - /// - /// Constructor for creating a local audio track. - /// - /// The newly created track wrapper. - /// The settings to initialize the newly created native track. - /// - public LocalAudioTrackInteropInitConfig(LocalAudioTrack track, LocalAudioTrackSettings settings) - { - } - } - /// /// Helper structure to pass parameters to the native implementation when creating a local video track /// by opening a local video capture device. diff --git a/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs b/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs index e1821dbc8..c2953c2a3 100644 --- a/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs +++ b/libs/Microsoft.MixedReality.WebRTC/LocalAudioTrack.cs @@ -12,7 +12,7 @@ namespace Microsoft.MixedReality.WebRTC /// /// Settings for adding a local audio track backed by a local audio capture device (e.g. microphone). /// - public class LocalAudioTrackSettings + public class LocalAudioTrackInitConfig { /// /// Name of the track to create, as used for the SDP negotiation. @@ -52,6 +52,11 @@ public bool Enabled } } + /// + /// Audio track source this track is pulling its audio frames from. + /// + public AudioTrackSource Source { get; private set; } = null; + /// /// Event that occurs when a audio frame has been produced by the underlying source and is available. /// @@ -77,55 +82,62 @@ public bool Enabled private LocalAudioTrackInterop.InteropCallbackArgs _interopCallbackArgs; /// - /// Create an audio track from a local audio capture device (microphone). + /// Create an audio track from an existing audio track source. + /// /// This does not add the track to any peer connection. Instead, the track must be added manually to /// an audio transceiver to be attached to a peer connection and transmitted to a remote peer. /// - /// Settings to initialize the local audio track. - /// Asynchronous task completed once the device is capturing and the track is created. - /// - /// On UWP this requires the "microphone" capability. - /// See - /// for more details. - /// - public static Task CreateFromDeviceAsync(LocalAudioTrackSettings settings = null) + /// The track source which provides the raw audio frames to the newly created track. + /// Configuration to initialize the track being created. + /// Asynchronous task completed once the track is created. + public static Task CreateFromSourceAsync(AudioTrackSource source, LocalAudioTrackInitConfig initConfig) { - return Task.Run(() => + if (source == null) { - // On UWP this cannot be called from the main UI thread, so always call it from - // a background worker thread. + throw new ArgumentNullException(); + } - string trackName = settings?.trackName; + return Task.Run(() => + { + // Parse and marshal the settings + string trackName = initConfig?.trackName; if (string.IsNullOrEmpty(trackName)) { trackName = Guid.NewGuid().ToString(); } + var config = new LocalAudioTrackInterop.TrackInitConfig + { + TrackName = trackName + }; // Create interop wrappers var track = new LocalAudioTrack(trackName); - // Parse settings - var config = new PeerConnectionInterop.LocalAudioTrackInteropInitConfig(track, settings); - // Create native implementation objects - uint res = LocalAudioTrackInterop.LocalAudioTrack_CreateFromDevice(config, trackName, - out LocalAudioTrackHandle trackHandle); + uint res = LocalAudioTrackInterop.LocalAudioTrack_CreateFromSource(in config, + source._nativeHandle, out LocalAudioTrackHandle trackHandle); Utils.ThrowOnErrorCode(res); - track.SetHandle(trackHandle); + + // Finish creating the track, and bind it to the source + track.FinishCreate(trackHandle, source); + source.OnTrackAddedToSource(track); + return track; }); } - // Constructor for interop-based creation; SetHandle() will be called later. + // Constructor for interop-based creation; FinishCreate() will be called later. // Constructor for standalone track not associated to a peer connection. - internal LocalAudioTrack(string trackName) : base(null, trackName) + internal LocalAudioTrack(string trackName) + : base(null, trackName) { Transceiver = null; } - // Constructor for interop-based creation; SetHandle() will be called later. + // Constructor for interop-based creation; FinishCreate() will be called later. // Constructor for a track associated with a peer connection. - internal LocalAudioTrack(PeerConnection peer, Transceiver transceiver, string trackName) : base(peer, trackName) + internal LocalAudioTrack(PeerConnection peer, Transceiver transceiver, string trackName) + : base(peer, trackName) { Debug.Assert(transceiver.MediaKind == MediaKind.Audio); Debug.Assert(transceiver.LocalAudioTrack == null); @@ -133,7 +145,7 @@ internal LocalAudioTrack(PeerConnection peer, Transceiver transceiver, string tr transceiver.LocalAudioTrack = this; } - internal void SetHandle(LocalAudioTrackHandle handle) + internal void FinishCreate(LocalAudioTrackHandle handle, AudioTrackSource source) { Debug.Assert(!handle.IsClosed); // Either first-time assign or no-op (assign same value again) @@ -143,6 +155,7 @@ internal void SetHandle(LocalAudioTrackHandle handle) _nativeHandle = handle; RegisterInteropCallbacks(); } + Source = source; } private void RegisterInteropCallbacks() @@ -175,6 +188,13 @@ public void Dispose() Debug.Assert(PeerConnection == null); Debug.Assert(Transceiver == null); + // Notify the source + if (Source != null) + { + Source.OnTrackRemovedFromSource(this); + Source = null; + } + // Unregister interop callbacks if (_selfHandle != IntPtr.Zero) { diff --git a/libs/mrwebrtc/include/audio_track_source_interop.h b/libs/mrwebrtc/include/audio_track_source_interop.h new file mode 100644 index 000000000..77cb05c4d --- /dev/null +++ b/libs/mrwebrtc/include/audio_track_source_interop.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "interop_api.h" + +extern "C" { + +/// Configuration for opening a local audio capture device (microphone) as an +/// audio track source. +struct mrsLocalAudioDeviceInitConfig { + /// Enable auto gain control (AGC). + mrsOptBool auto_gain_control_{mrsOptBool::kUnset}; +}; + +/// Add a reference to the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTrackSourceAddRef(mrsAudioTrackSourceHandle handle) noexcept; + +/// Remove a reference from the native object associated with the given handle. +MRS_API void MRS_CALL +mrsAudioTrackSourceRemoveRef(mrsAudioTrackSourceHandle handle) noexcept; + +/// Assign some name to the track source, for logging and debugging. +MRS_API void MRS_CALL +mrsAudioTrackSourceSetName(mrsAudioTrackSourceHandle handle, + const char* name) noexcept; + +/// Get the name to the track source. The caller must provide a buffer with a +/// sufficent size to copy the name to, including a null terminator character. +/// The |buffer| argument points to the raw buffer, and the |buffer_size| to the +/// capacity of the buffer, in bytes. On return, if the buffer has enough +/// capacity for the name and its null terminator, the name is copied to the +/// buffer, and the actual buffer size consumed (including null terminator) is +/// returned in |buffer_size|. If not, then the function returns +/// |mrsResult::kBufferTooSmall|, and |buffer_size| contains the total size that +/// the buffer would need for the call to succeed, such that the caller can +/// retry with a buffer with that capacity. +MRS_API mrsResult MRS_CALL +mrsAudioTrackSourceGetName(mrsAudioTrackSourceHandle handle, + char* buffer, + uint64_t* buffer_size) noexcept; + +/// Assign some opaque user data to the audio track source. The implementation +/// will store the pointer in the audio track source object and not touch it. It +/// can be retrieved with |mrsAudioTrackSourceGetUserData()| at any point during +/// the object lifetime. This is not multithread-safe. +MRS_API void MRS_CALL +mrsAudioTrackSourceSetUserData(mrsAudioTrackSourceHandle handle, + void* user_data) noexcept; + +/// Get the opaque user data pointer previously assigned to the audio track +/// source with |mrsAudioTrackSourceSetUserData()|. If no value was previously +/// assigned, return |nullptr|. This is not multithread-safe. +MRS_API void* MRS_CALL +mrsAudioTrackSourceGetUserData(mrsAudioTrackSourceHandle handle) noexcept; + +/// Create an audio track source by opening a local audio capture device +/// (microphone). +MRS_API mrsResult MRS_CALL mrsAudioTrackSourceCreateFromDevice( + const mrsLocalAudioDeviceInitConfig* init_config, + mrsAudioTrackSourceHandle* source_handle_out) noexcept; + +/// Register a custom callback to be called when the audio track source produced +/// a frame. +/// WARNING: The default platform source internal implementation currently does +/// not hook those callbacks, and therefore the callback will never be called. +/// This is a limitation of the underlying implementation. +/// See https://bugs.chromium.org/p/webrtc/issues/detail?id=11602 +MRS_API void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( + mrsAudioTrackSourceHandle source_handle, + mrsAudioFrameCallback callback, + void* user_data) noexcept + +} // extern "C" diff --git a/libs/mrwebrtc/include/interop_api.h b/libs/mrwebrtc/include/interop_api.h index 9df8944a1..a41f95f40 100644 --- a/libs/mrwebrtc/include/interop_api.h +++ b/libs/mrwebrtc/include/interop_api.h @@ -89,6 +89,9 @@ using mrsMediaTrackHandle = void*; /// Opaque handle to a native Transceiver interop object. using mrsTransceiverHandle = void*; +/// Opaque handle to a native AudioTrackSource interop object. +using mrsAudioTrackSourceHandle = void*; + /// Opaque handle to a native LocalAudioTrack interop object. using mrsLocalAudioTrackHandle = void*; diff --git a/libs/mrwebrtc/include/local_audio_track_interop.h b/libs/mrwebrtc/include/local_audio_track_interop.h index e449d7a9b..3b7687629 100644 --- a/libs/mrwebrtc/include/local_audio_track_interop.h +++ b/libs/mrwebrtc/include/local_audio_track_interop.h @@ -4,14 +4,18 @@ #pragma once #include "audio_frame_observer.h" +#include "audio_track_source_interop.h" #include "export.h" #include "interop_api.h" extern "C" { -/// Configuration for opening a local audio capture device and creating a local -/// audio track. -struct mrsLocalAudioTrackInitConfig {}; +/// Configuration for creating a local audio track. +struct mrsLocalAudioTrackInitSettings { + /// Track name. This must be a valid SDP token (see |mrsSdpTokenIsValid()|), or + /// |nullptr| to let the implementation generate a valid unique track name. + const char* track_name{}; +}; /// Add a reference to the native object associated with the given handle. MRS_API void MRS_CALL @@ -21,11 +25,10 @@ mrsLocalAudioTrackAddRef(mrsLocalAudioTrackHandle handle) noexcept; MRS_API void MRS_CALL mrsLocalAudioTrackRemoveRef(mrsLocalAudioTrackHandle handle) noexcept; -/// Create a new local audio track by opening a local audio capture device -/// (webcam). -MRS_API mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( - const mrsLocalAudioTrackInitConfig* config, - const char* track_name, +/// Create a new local audio track from an audio track source. +MRS_API mrsResult MRS_CALL mrsLocalAudioTrackCreateFromSource( + const mrsLocalAudioTrackInitSettings* init_settings, + mrsAudioTrackSourceHandle source_handle, mrsLocalAudioTrackHandle* track_handle_out) noexcept; /// Register a custom callback to be called when the local audio track captured diff --git a/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp b/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp new file mode 100644 index 000000000..24473b94c --- /dev/null +++ b/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is a precompiled header, it must be on its own, followed by a blank +// line, to prevent clang-format from reordering it with other headers. +#include "pch.h" + +#include "audio_track_source_interop.h" +#include "callback.h" +#include "global_factory.h" +#include "media/audio_track_source.h" +#include "utils.h" + +using namespace Microsoft::MixedReality::WebRTC; + +void MRS_CALL +mrsAudioTrackSourceAddRef(mrsAudioTrackSourceHandle handle) noexcept { + if (auto source = static_cast(handle)) { + source->AddRef(); + } else { + RTC_LOG(LS_WARNING) + << "Trying to add reference to NULL AudioTrackSource object."; + } +} + +void MRS_CALL +mrsAudioTrackSourceRemoveRef(mrsAudioTrackSourceHandle handle) noexcept { + if (auto source = static_cast(handle)) { + source->RemoveRef(); + } else { + RTC_LOG(LS_WARNING) << "Trying to remove reference from NULL " + "AudioTrackSource object."; + } +} + +void MRS_CALL mrsAudioTrackSourceSetName(mrsAudioTrackSourceHandle handle, + const char* name) noexcept { + if (auto source = static_cast(handle)) { + source->SetName(name); + } +} + +mrsResult MRS_CALL mrsAudioTrackSourceGetName(mrsAudioTrackSourceHandle handle, + char* buffer, + uint64_t* buffer_size) noexcept { + auto source = static_cast(handle); + if (!source) { + RTC_LOG(LS_ERROR) << "Invalid handle to audio track source."; + return mrsResult::kInvalidNativeHandle; + } + if (!buffer) { + RTC_LOG(LS_ERROR) << "Invalid NULL string buffer."; + return mrsResult::kInvalidParameter; + } + if (!buffer_size) { + RTC_LOG(LS_ERROR) << "Invalid NULL string buffer size reference."; + return mrsResult::kInvalidParameter; + } + const std::string name = source->GetName(); + const size_t capacity = *buffer_size; + const size_t size_with_terminator = name.size() + 1; + // Always assign size, even if buffer too small + *buffer_size = size_with_terminator; + if (size_with_terminator <= capacity) { + memcpy(buffer, name.c_str(), size_with_terminator); + return mrsResult::kSuccess; + } + return mrsResult::kBufferTooSmall; +} + +void MRS_CALL mrsAudioTrackSourceSetUserData(mrsAudioTrackSourceHandle handle, + void* user_data) noexcept { + if (auto source = static_cast(handle)) { + source->SetUserData(user_data); + } +} + +void* MRS_CALL +mrsAudioTrackSourceGetUserData(mrsAudioTrackSourceHandle handle) noexcept { + if (auto source = static_cast(handle)) { + return source->GetUserData(); + } + return nullptr; +} + +mrsResult MRS_CALL mrsAudioTrackSourceCreateFromDevice( + const mrsLocalAudioDeviceInitConfig* init_config, + mrsAudioTrackSourceHandle* source_handle_out) noexcept { + if (!source_handle_out) { + RTC_LOG(LS_ERROR) << "Invalid NULL audio track source handle."; + return Result::kInvalidParameter; + } + *source_handle_out = nullptr; + + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kInvalidOperation; + } + + // Create the audio track source + cricket::AudioOptions options{}; + options.auto_gain_control = ToOptional(init_config->auto_gain_control_); + rtc::scoped_refptr audio_source = + pc_factory->CreateAudioSource(options); + if (!audio_source) { + RTC_LOG(LS_ERROR) + << "Failed to create audio source from local audio capture device."; + return Result::kUnknownError; + } + + // Create the wrapper + RefPtr wrapper = + new AudioTrackSource(global_factory, std::move(audio_source)); + if (!wrapper) { + RTC_LOG(LS_ERROR) << "Failed to create audio track source."; + return Result::kUnknownError; + } + *source_handle_out = wrapper.release(); + return Result::kSuccess; +} + +void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( + mrsAudioTrackSourceHandle source_handle, + mrsAudioFrameCallback callback, + void* user_data) noexcept { + if (auto source = static_cast(source_handle)) { + source->SetCallback(AudioFrameReadyCallback{callback, user_data}); + } +} diff --git a/libs/mrwebrtc/src/interop/global_factory.cpp b/libs/mrwebrtc/src/interop/global_factory.cpp index 24d879a9e..d655394d4 100644 --- a/libs/mrwebrtc/src/interop/global_factory.cpp +++ b/libs/mrwebrtc/src/interop/global_factory.cpp @@ -40,6 +40,8 @@ absl::string_view ObjectTypeToString(ObjectType type) { return "AudioTransceiver"; case ObjectType::kVideoTransceiver: return "VideoTransceiver"; + case ObjectType::kAudioTrackSource: + return "AudioTrackSource"; default: RTC_NOTREACHED(); return ""; diff --git a/libs/mrwebrtc/src/interop/interop_api.cpp b/libs/mrwebrtc/src/interop/interop_api.cpp index ab6ad8085..64875cb88 100644 --- a/libs/mrwebrtc/src/interop/interop_api.cpp +++ b/libs/mrwebrtc/src/interop/interop_api.cpp @@ -7,6 +7,7 @@ #include "api/stats/rtcstats_objects.h" +#include "audio_track_source_interop.h" #include "data_channel.h" #include "data_channel_interop.h" #include "external_video_track_source_interop.h" @@ -14,6 +15,7 @@ #include "interop_api.h" #include "local_audio_track_interop.h" #include "local_video_track_interop.h" +#include "media/audio_track_source.h" #include "media/external_video_track_source_impl.h" #include "media/local_audio_track.h" #include "media/local_video_track.h" @@ -684,49 +686,6 @@ void MRS_CALL mrsPeerConnectionRegisterDataChannelRemovedCallback( } } -mrsResult MRS_CALL mrsLocalAudioTrackCreateFromDevice( - const mrsLocalAudioTrackInitConfig* /*config*/, - const char* track_name, - mrsLocalAudioTrackHandle* track_handle_out) noexcept { - if (IsStringNullOrEmpty(track_name)) { - RTC_LOG(LS_ERROR) << "Invalid empty local audio track name."; - return Result::kInvalidParameter; - } - if (!track_handle_out) { - RTC_LOG(LS_ERROR) << "Invalid NULL local audio track handle."; - return Result::kInvalidParameter; - } - *track_handle_out = nullptr; - - RefPtr global_factory(GlobalFactory::InstancePtr()); - auto pc_factory = global_factory->GetPeerConnectionFactory(); - if (!pc_factory) { - return Result::kInvalidOperation; - } - - // Create the audio track source - rtc::scoped_refptr audio_source = - pc_factory->CreateAudioSource(cricket::AudioOptions()); - if (!audio_source) { - RTC_LOG(LS_ERROR) << "Failed to create local audio source."; - return Result::kUnknownError; - } - - // Create the audio track - rtc::scoped_refptr audio_track = - pc_factory->CreateAudioTrack(track_name, audio_source); - if (!audio_track) { - RTC_LOG(LS_ERROR) << "Failed to create local audio track."; - return Result::kUnknownError; - } - - // Create the audio track wrapper - RefPtr track = - new LocalAudioTrack(std::move(global_factory), std::move(audio_track)); - *track_handle_out = track.release(); - return Result::kSuccess; -} - mrsResult MRS_CALL mrsLocalVideoTrackCreateFromDevice( const mrsLocalVideoTrackInitConfig* config, const char* track_name, diff --git a/libs/mrwebrtc/src/interop/local_audio_track_interop.cpp b/libs/mrwebrtc/src/interop/local_audio_track_interop.cpp index cadec5ab3..2f06daa25 100644 --- a/libs/mrwebrtc/src/interop/local_audio_track_interop.cpp +++ b/libs/mrwebrtc/src/interop/local_audio_track_interop.cpp @@ -5,8 +5,11 @@ // line, to prevent clang-format from reordering it with other headers. #include "pch.h" +#include "interop/global_factory.h" #include "local_audio_track_interop.h" +#include "media/audio_track_source.h" #include "media/local_audio_track.h" +#include "utils.h" using namespace Microsoft::MixedReality::WebRTC; @@ -34,7 +37,51 @@ mrsLocalAudioTrackRemoveRef(mrsLocalAudioTrackHandle handle) noexcept { } } -// mrsLocalAudioTrackCreateFromDevice -> interop_api.cpp +mrsResult MRS_CALL mrsLocalAudioTrackCreateFromSource( + const mrsLocalAudioTrackInitSettings* init_settings, + mrsAudioTrackSourceHandle source_handle, + mrsLocalAudioTrackHandle* track_handle_out) noexcept { + if (!init_settings) { + RTC_LOG(LS_ERROR) << "Invalid NULL local audio track init settings."; + return Result::kInvalidParameter; + } + if (mrsBool::kFalse == mrsSdpIsValidToken(init_settings->track_name)) { + RTC_LOG(LS_ERROR) << "Local audio track name " << init_settings->track_name + << " is not a valid SDP token."; + return Result::kInvalidParameter; + } + if (!source_handle) { + RTC_LOG(LS_ERROR) << "Invalid track source handle."; + return Result::kInvalidParameter; + } + if (!track_handle_out) { + RTC_LOG(LS_ERROR) << "Invalid NULL local audio track handle reference."; + return Result::kInvalidParameter; + } + *track_handle_out = nullptr; + + RefPtr global_factory(GlobalFactory::InstancePtr()); + auto pc_factory = global_factory->GetPeerConnectionFactory(); + if (!pc_factory) { + return Result::kInvalidOperation; + } + + // Create the audio track + auto source = static_cast(source_handle); + rtc::scoped_refptr audio_track = + pc_factory->CreateAudioTrack(init_settings->track_name, + source->impl().get()); + if (!audio_track) { + RTC_LOG(LS_ERROR) << "Failed to create local audio track from source."; + return Result::kUnknownError; + } + + // Create the audio track wrapper + RefPtr track = + new LocalAudioTrack(std::move(global_factory), std::move(audio_track)); + *track_handle_out = track.release(); + return Result::kSuccess; +} void MRS_CALL mrsLocalAudioTrackRegisterFrameCallback(mrsLocalAudioTrackHandle trackHandle, diff --git a/libs/mrwebrtc/src/media/audio_track_source.cpp b/libs/mrwebrtc/src/media/audio_track_source.cpp new file mode 100644 index 000000000..2cc2095e4 --- /dev/null +++ b/libs/mrwebrtc/src/media/audio_track_source.cpp @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "interop/global_factory.h" +#include "media/audio_track_source.h" + +namespace Microsoft { +namespace MixedReality { +namespace WebRTC { + +AudioSourceAdapter::AudioSourceAdapter( + rtc::scoped_refptr source) + : source_(std::move(source)), state_(source->state()) {} + +void AudioSourceAdapter::RegisterObserver(webrtc::ObserverInterface* observer) { + observer_ = observer; +} + +void AudioSourceAdapter::UnregisterObserver( + webrtc::ObserverInterface* observer) { + RTC_DCHECK_EQ(observer_, observer); + observer_ = nullptr; +} + +void AudioSourceAdapter::AddSink(webrtc::AudioTrackSinkInterface* sink) { + sinks_.push_back(sink); +} + +void AudioSourceAdapter::RemoveSink(webrtc::AudioTrackSinkInterface* sink) { + auto it = std::find(sinks_.begin(), sinks_.end(), sink); + if (it != sinks_.end()) { + sinks_.erase(it); + } +} + +AudioTrackSource::AudioTrackSource( + RefPtr global_factory, + rtc::scoped_refptr source) noexcept + : TrackedObject(std::move(global_factory), ObjectType::kAudioTrackSource), + source_(std::move(source)) { + RTC_CHECK(source_); +} + +AudioTrackSource::~AudioTrackSource() { + if (observer_) { + source_->RemoveSink(observer_.get()); + } +} + +void AudioTrackSource::SetCallback(AudioFrameReadyCallback callback) noexcept { + std::lock_guard lock(observer_mutex_); + if (callback) { + // When assigning a new callback, create and register an observer. + if (!observer_) { + observer_ = std::make_unique(); + source_->AddSink(observer_.get()); + } + observer_->SetCallback(callback); + } else { + // When clearing the existing callback, unregister and destroy the observer. + // This ensures the native source knows when there is no more observer, and + // can potentially optimize its behavior. + if (observer_) { + source_->RemoveSink(observer_.get()); + observer_.reset(); + } + } +} + +} // namespace WebRTC +} // namespace MixedReality +} // namespace Microsoft diff --git a/libs/mrwebrtc/src/media/audio_track_source.h b/libs/mrwebrtc/src/media/audio_track_source.h new file mode 100644 index 000000000..6c6abd157 --- /dev/null +++ b/libs/mrwebrtc/src/media/audio_track_source.h @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "external_video_track_source_interop.h" +#include "mrs_errors.h" +#include "refptr.h" +#include "tracked_object.h" +#include "video_frame.h" + +#include "api/mediastreaminterface.h" + +namespace Microsoft { +namespace MixedReality { +namespace WebRTC { + +class AudioTrackSource; + +/// Adapter for a local audio source backing one or more local audio tracks. +class AudioSourceAdapter : public webrtc::AudioSourceInterface { + public: + AudioSourceAdapter(rtc::scoped_refptr source); + + // + // NotifierInterface + // + + void RegisterObserver(webrtc::ObserverInterface* observer) override; + void UnregisterObserver(webrtc::ObserverInterface* observer) override; + + // + // MediaSourceInterface + // + + SourceState state() const override { return state_; } + bool remote() const override { return false; } + + // + // AudioSourceInterface + // + + // Sets the volume of the source. |volume| is in the range of [0, 10]. + // TODO(tommi): This method should be on the track and ideally volume should + // be applied in the track in a way that does not affect clones of the track. + void SetVolume(double /*volume*/) override {} + + // Registers/unregisters observers to the audio source. + void RegisterAudioObserver(AudioObserver* /*observer*/) override {} + void UnregisterAudioObserver(AudioObserver* /*observer*/) override {} + + // TODO(tommi): Make pure virtual. + void AddSink(webrtc::AudioTrackSinkInterface* sink) override; + void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override; + + protected: + rtc::scoped_refptr source_; + std::vector sinks_; + SourceState state_{SourceState::kEnded}; + webrtc::ObserverInterface* observer_{nullptr}; + AudioObserver* audio_observer_{nullptr}; +}; + +/// Base class for an audio track source acting as a frame source for one or +/// more audio tracks. +class AudioTrackSource : public TrackedObject { + public: + ///// Helper to create an audio track source from a custom audio frame request + ///// callback. + // static RefPtr createFromCustom( + // RefPtr global_factory, + // RefPtr audio_source); + + AudioTrackSource( + RefPtr global_factory, + rtc::scoped_refptr source) noexcept; + ~AudioTrackSource() override; + + void SetName(absl::string_view name) { + name_.assign(name.data(), name.size()); + } + + /// Get the name of the audio track source. + std::string GetName() const noexcept override { return name_; } + + void SetCallback(AudioFrameReadyCallback callback) noexcept; + + inline rtc::scoped_refptr impl() const + noexcept { + return source_; + } + + protected: + rtc::scoped_refptr source_; + std::string name_; + std::unique_ptr observer_; + std::mutex observer_mutex_; +}; + +namespace detail { + +// +// Helpers +// + +///// Create a custom audio track source wrapping the given interop callback. +// RefPtr AudioTrackSourceCreateFromCustom( +// RefPtr global_factory, +// mrsRequestCustomAudioFrameCallback callback, +// void* user_data); + +} // namespace detail + +} // namespace WebRTC +} // namespace MixedReality +} // namespace Microsoft diff --git a/libs/mrwebrtc/src/tracked_object.h b/libs/mrwebrtc/src/tracked_object.h index 77b9f8aa0..bba829818 100644 --- a/libs/mrwebrtc/src/tracked_object.h +++ b/libs/mrwebrtc/src/tracked_object.h @@ -29,6 +29,7 @@ enum class ObjectType : int { kDataChannel, kAudioTransceiver, kVideoTransceiver, + kAudioTrackSource, }; /// Object tracked for interop, exposing helper methods for debugging purpose. diff --git a/libs/mrwebrtc/test/audio_track_source_tests.cpp b/libs/mrwebrtc/test/audio_track_source_tests.cpp new file mode 100644 index 000000000..5c644dc72 --- /dev/null +++ b/libs/mrwebrtc/test/audio_track_source_tests.cpp @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "pch.h" + +#include "audio_frame.h" +#include "audio_track_source_interop.h" +#include "interop_api.h" + +#include "test_utils.h" + +namespace { + +class AudioTrackSourceTests + : public TestUtils::TestBase, + public testing::WithParamInterface {}; + +} // namespace + +#if !defined(MRSW_EXCLUDE_DEVICE_TESTS) + +INSTANTIATE_TEST_CASE_P(, + AudioTrackSourceTests, + testing::ValuesIn(TestUtils::TestSemantics), + TestUtils::SdpSemanticToString); + +TEST_P(AudioTrackSourceTests, CreateFromDevice) { + mrsLocalAudioDeviceInitConfig config{}; + mrsAudioTrackSourceHandle source_handle{}; + ASSERT_EQ(mrsResult::kSuccess, + mrsAudioTrackSourceCreateFromDevice(&config, &source_handle)); + ASSERT_NE(nullptr, source_handle); + mrsAudioTrackSourceRemoveRef(source_handle); +} + +// TODO - Don't use device, run outside MRSW_EXCLUDE_DEVICE_TESTS +TEST_P(AudioTrackSourceTests, Name) { + mrsLocalAudioDeviceInitConfig config{}; + mrsAudioTrackSourceHandle source_handle{}; + ASSERT_EQ(mrsResult::kSuccess, + mrsAudioTrackSourceCreateFromDevice(&config, &source_handle)); + ASSERT_NE(nullptr, source_handle); + + // Exact-fit buffer + { + constexpr const char kTestName[] = "test_name_exact_fit_buffer"; + constexpr const size_t kTestNameSize = + sizeof(kTestName); // incl. zero-terminator + mrsAudioTrackSourceSetName(source_handle, kTestName); + constexpr const size_t kBufferSize = kTestNameSize; // exact-fit + char buffer[kBufferSize]; + size_t size = kBufferSize; + ASSERT_EQ(mrsResult::kSuccess, + mrsAudioTrackSourceGetName(source_handle, buffer, &size)); + ASSERT_EQ(kTestNameSize, size); + ASSERT_EQ(0, memcmp(buffer, kTestName, kTestNameSize)); + } + + // Larger buffer + { + constexpr const char kTestName[] = "test_name_larger_buffer"; + constexpr const size_t kTestNameSize = + sizeof(kTestName); // incl. zero-terminator + mrsAudioTrackSourceSetName(source_handle, kTestName); + constexpr const size_t kBufferSize = kTestNameSize + 1; // larger + char buffer[kBufferSize]; + size_t size = kBufferSize; + ASSERT_EQ(mrsResult::kSuccess, + mrsAudioTrackSourceGetName(source_handle, buffer, &size)); + ASSERT_EQ(kTestNameSize, size); + ASSERT_EQ(0, memcmp(buffer, kTestName, kTestNameSize)); + } + + // Buffer too small + { + constexpr const char kTestName[] = "test_name_buffer_too_small"; + constexpr const size_t kTestNameSize = + sizeof(kTestName); // incl. zero-terminator + mrsAudioTrackSourceSetName(source_handle, kTestName); + constexpr const size_t kBufferSize = kTestNameSize - 1; // too small + char buffer[kBufferSize]; + size_t size = kBufferSize; + ASSERT_EQ(mrsResult::kBufferTooSmall, + mrsAudioTrackSourceGetName(source_handle, buffer, &size)); + ASSERT_EQ(kTestNameSize, size); + } + + // Invalid buffer + { + size_t size = 0; + ASSERT_EQ(mrsResult::kInvalidParameter, + mrsAudioTrackSourceGetName(source_handle, nullptr, &size)); + } + + // Invalid size + { + char buffer[5]; + ASSERT_EQ(mrsResult::kInvalidParameter, + mrsAudioTrackSourceGetName(source_handle, buffer, nullptr)); + } + + mrsAudioTrackSourceRemoveRef(source_handle); +} + +#endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/libs/mrwebrtc/test/audio_track_tests.cpp b/libs/mrwebrtc/test/audio_track_tests.cpp index 7e5fc12d2..caeb3de8c 100644 --- a/libs/mrwebrtc/test/audio_track_tests.cpp +++ b/libs/mrwebrtc/test/audio_track_tests.cpp @@ -127,11 +127,20 @@ TEST_P(AudioTrackTests, Simple) { &audio_transceiver1)); ASSERT_NE(nullptr, audio_transceiver1); + // Create the audio source #1 + mrsLocalAudioDeviceInitConfig device_config{}; + mrsAudioTrackSourceHandle audio_source1{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTrackSourceCreateFromDevice( + &device_config, &audio_source1)); + ASSERT_NE(nullptr, audio_source1); + // Create the local audio track #1 - mrsLocalAudioTrackInitConfig config{}; + mrsLocalAudioTrackInitSettings init_settings{}; + init_settings.track_name = "test_audio_track"; mrsLocalAudioTrackHandle audio_track1{}; - ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( - &config, "test_audio_track", &audio_track1)); + ASSERT_EQ(Result::kSuccess, + mrsLocalAudioTrackCreateFromSource(&init_settings, audio_source1, + &audio_track1)); ASSERT_NE(nullptr, audio_track1); // Audio tracks start enabled @@ -242,6 +251,7 @@ TEST_P(AudioTrackTests, Simple) { // Clean-up mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTrackSourceRemoveRef(audio_source1); } TEST_P(AudioTrackTests, Muted) { @@ -274,11 +284,20 @@ TEST_P(AudioTrackTests, Muted) { &audio_transceiver1)); ASSERT_NE(nullptr, audio_transceiver1); + // Create the audio source #1 + mrsLocalAudioDeviceInitConfig device_config{}; + mrsAudioTrackSourceHandle audio_source1{}; + ASSERT_EQ(Result::kSuccess, mrsAudioTrackSourceCreateFromDevice( + &device_config, &audio_source1)); + ASSERT_NE(nullptr, audio_source1); + // Create the local audio track #1 - mrsLocalAudioTrackInitConfig config{}; + mrsLocalAudioTrackInitSettings init_settings{}; + init_settings.track_name = "test_audio_track"; mrsLocalAudioTrackHandle audio_track1{}; - ASSERT_EQ(Result::kSuccess, mrsLocalAudioTrackCreateFromDevice( - &config, "test_audio_track", &audio_track1)); + ASSERT_EQ(Result::kSuccess, + mrsLocalAudioTrackCreateFromSource(&init_settings, audio_source1, + &audio_track1)); ASSERT_NE(nullptr, audio_track1); // Audio tracks start enabled @@ -377,6 +396,7 @@ TEST_P(AudioTrackTests, Muted) { // Clean-up mrsRemoteAudioTrackRegisterFrameCallback(audio_track2, nullptr, nullptr); mrsLocalAudioTrackRemoveRef(audio_track1); + mrsAudioTrackSourceRemoveRef(audio_source1); } #endif // MRSW_EXCLUDE_DEVICE_TESTS diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/AudioTrackSourceTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/AudioTrackSourceTests.cs new file mode 100644 index 000000000..a073c3cb0 --- /dev/null +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/AudioTrackSourceTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using NUnit.Framework; +using NUnit.Framework.Internal; + +namespace Microsoft.MixedReality.WebRTC.Tests +{ + [TestFixture(SdpSemantic.PlanB)] + [TestFixture(SdpSemantic.UnifiedPlan)] + internal class AudioTrackSourceTests : PeerConnectionTestBase + { + public AudioTrackSourceTests(SdpSemantic sdpSemantic) : base(sdpSemantic) + { + } + +#if !MRSW_EXCLUDE_DEVICE_TESTS + + [Test] + public async Task CreateFromDevice() + { + using (AudioTrackSource source = await AudioTrackSource.CreateFromDeviceAsync()) + { + Assert.IsNotNull(source); + Assert.AreEqual(string.Empty, source.Name); + Assert.AreEqual(0, source.Tracks.Count); + } + } + + // TODO - Don't use device, run outside MRSW_EXCLUDE_DEVICE_TESTS + [Test] + public async Task Name() + { + using (AudioTrackSource source = await AudioTrackSource.CreateFromDeviceAsync()) + { + Assert.IsNotNull(source); + + const string kTestName = "test_audio_track_source_name"; + source.Name = kTestName; + Assert.AreEqual(kTestName, source.Name); + } + } + +#endif // MRSW_EXCLUDE_DEVICE_TESTS + } +} diff --git a/tests/Microsoft.MixedReality.WebRTC.Tests/LocalAudioTrackTests.cs b/tests/Microsoft.MixedReality.WebRTC.Tests/LocalAudioTrackTests.cs new file mode 100644 index 000000000..b19d7e87a --- /dev/null +++ b/tests/Microsoft.MixedReality.WebRTC.Tests/LocalAudioTrackTests.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using NUnit.Framework; +using NUnit.Framework.Internal; + +namespace Microsoft.MixedReality.WebRTC.Tests +{ + [TestFixture(SdpSemantic.PlanB)] + [TestFixture(SdpSemantic.UnifiedPlan)] + internal class LocalAudioTrackTests : PeerConnectionTestBase + { + public LocalAudioTrackTests(SdpSemantic sdpSemantic) : base(sdpSemantic) + { + } + +#if !MRSW_EXCLUDE_DEVICE_TESTS + + // TODO - Don't use device, run outside MRSW_EXCLUDE_DEVICE_TESTS + [Test] + public async Task CreateFromSource() + { + using (AudioTrackSource source = await AudioTrackSource.CreateFromDeviceAsync()) + { + Assert.IsNotNull(source); + + var settings = new LocalAudioTrackInitConfig { trackName = "track_name" }; + using (LocalAudioTrack track = await LocalAudioTrack.CreateFromSourceAsync(source, settings)) + { + Assert.IsNotNull(track); + } + } + } + +#endif // MRSW_EXCLUDE_DEVICE_TESTS + } +} diff --git a/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj b/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj index fe986843b..f7d50a3d2 100644 --- a/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj +++ b/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj @@ -1,249 +1,253 @@ - - - - - - - - - - - - - - - - - - Debug - ARM - - - Debug - Win32 - - - Debug - x64 - - - Release - ARM - - - Release - Win32 - - - Release - x64 - - - - {0a6c45b9-76a4-4ab6-947b-0c0519aae67d} - DynamicLibrary - Microsoft_MixedReality_WebRTC - en-US - 14.0 - true - Windows Store - 10.0.17763.0 - 10.0.17763.0 - 10.0 - mrwebrtc-uwp - - - - DynamicLibrary - v141 - Unicode - - - true - - - false - true - - - - - - - - - - - - - false - true - $(MRWebRTCProjectRoot)bin\UWP\$(PlatformTarget)\$(Configuration)\ - $(MRWebRTCProjectRoot)build\mrwebrtc\UWP\$(PlatformTarget)\$(Configuration)\ - mrwebrtc - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\x86 - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\x86_64 - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\ARM - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\ARM64 - - - true - - - false - - - - - - - _WINDLL;MR_SHARING_WIN;MRS_USE_STR_WRAPPER;WINUWP;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;%(PreprocessorDefinitions) - Use - false - Level4 - true - $(MRWebRTCProjectRoot)libs\mrwebrtc\include;$(MRWebRTCProjectRoot)libs\mrwebrtc\src;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc;$(WebRTCCoreRepoPath)webrtc\xplatform;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\generated\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\override\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium\third_party\abseil-cpp;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\third_party\idl;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib-eventing;$(WebRTCCoreRepoPath)webrtc\xplatform\libyuv\include;%(AdditionalIncludeDirectories) - false - stdcpp17 - SyncCThrow - /Zc:__cplusplus %(AdditionalOptions) - - - Console - - - false - Ole32.lib;Evr.lib;mf.lib;mfuuid.lib;mfplat.lib;webrtc.lib;WindowsApp.lib;Org.WebRtc.WrapperGlue.lib;%(AdditionalDependencies) - $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\winuwp\$(PlatformTarget)\$(Configuration);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.WrapperGlue.Universal\Build\Output\Org.WebRtc.WrapperGlue\$(Configuration)\$(PlatformTarget);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.Universal\Build\Output\Org.WebRtc\$(Configuration)\$(PlatformTarget);%(AdditionalLibraryDirectories) - DebugFull - - - - - Disabled - _DEBUG;%(PreprocessorDefinitions) - MultiThreadedDebugDLL - - - - - MaxSpeed - true - true - NDEBUG;%(PreprocessorDefinitions) - true - MultiThreadedDLL - - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + Debug + ARM + + + Debug + Win32 + + + Debug + x64 + + + Release + ARM + + + Release + Win32 + + + Release + x64 + + + + {0a6c45b9-76a4-4ab6-947b-0c0519aae67d} + DynamicLibrary + Microsoft_MixedReality_WebRTC + en-US + 14.0 + true + Windows Store + 10.0.17763.0 + 10.0.17763.0 + 10.0 + mrwebrtc-uwp + + + + DynamicLibrary + v141 + Unicode + + + true + + + false + true + + + + + + + + + + + + + false + true + $(MRWebRTCProjectRoot)bin\UWP\$(PlatformTarget)\$(Configuration)\ + $(MRWebRTCProjectRoot)build\mrwebrtc\UWP\$(PlatformTarget)\$(Configuration)\ + mrwebrtc + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\x86 + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\x86_64 + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\ARM + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\WSA\ARM64 + + + true + + + false + + + + + + + _WINDLL;MR_SHARING_WIN;MRS_USE_STR_WRAPPER;WINUWP;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;%(PreprocessorDefinitions) + Use + false + Level4 + true + $(MRWebRTCProjectRoot)libs\mrwebrtc\include;$(MRWebRTCProjectRoot)libs\mrwebrtc\src;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc;$(WebRTCCoreRepoPath)webrtc\xplatform;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\generated\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\override\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium\third_party\abseil-cpp;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\third_party\idl;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib-eventing;$(WebRTCCoreRepoPath)webrtc\xplatform\libyuv\include;%(AdditionalIncludeDirectories) + false + stdcpp17 + SyncCThrow + /Zc:__cplusplus %(AdditionalOptions) + + + Console + + + false + Ole32.lib;Evr.lib;mf.lib;mfuuid.lib;mfplat.lib;webrtc.lib;WindowsApp.lib;Org.WebRtc.WrapperGlue.lib;%(AdditionalDependencies) + $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\winuwp\$(PlatformTarget)\$(Configuration);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.WrapperGlue.Universal\Build\Output\Org.WebRtc.WrapperGlue\$(Configuration)\$(PlatformTarget);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.Universal\Build\Output\Org.WebRtc\$(Configuration)\$(PlatformTarget);%(AdditionalLibraryDirectories) + DebugFull + + + + + Disabled + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + + + + + MaxSpeed + true + true + NDEBUG;%(PreprocessorDefinitions) + true + MultiThreadedDLL + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Create + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj.filters b/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj.filters index d645ecd23..f0a8a979b 100644 --- a/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj.filters +++ b/tools/build/mrwebrtc/uwp/mrwebrtc-uwp.vcxproj.filters @@ -1,226 +1,238 @@ - - - - - src\interop - - - src\interop - - - src\interop - - - src\interop - - - src\interop - - - src\media - - - src\media - - - src\media - - - src\media - - - src\interop - - - src\interop - - - src\interop - - - src\media - - - src\media - - - src\interop - - - src\media - - - src\media - - - src\media - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src\interop - - - src - - - src\media - - - src - - - - - src\interop - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - include - - - src\media - - - include - - - include - - - src\media - - - src\media - - - include - - - include - - - include - - - include - - - include - - - include - - - include - - - include - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - include - - - src\media - - - src - - - - - - - - - {eed36cd1-e71f-4f6b-8eab-ef97af8acecf} - - - {079ac0ec-1e79-4f08-9773-86b5e28f7cb0} - - - {3716966c-d0c2-4ecc-a361-12fea77b532b} - - - {2c834693-8aec-4ee1-9c56-b8827db37535} - - + + + + + src\interop + + + src\interop + + + src\interop + + + src\interop + + + src\interop + + + src\media + + + src\media + + + src\media + + + src\media + + + src\interop + + + src\interop + + + src\interop + + + src\media + + + src\media + + + src\interop + + + src\media + + + src\media + + + src\media + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src\interop + + + src + + + src\media + + + src + + + src\media + + + src\interop + + + + + src\interop + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + include + + + src\media + + + include + + + include + + + src\media + + + src\media + + + include + + + include + + + include + + + include + + + include + + + include + + + include + + + include + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + include + + + src\media + + + src + + + src\media + + + include + + + + + + + + + {eed36cd1-e71f-4f6b-8eab-ef97af8acecf} + + + {079ac0ec-1e79-4f08-9773-86b5e28f7cb0} + + + {3716966c-d0c2-4ecc-a361-12fea77b532b} + + + {2c834693-8aec-4ee1-9c56-b8827db37535} + + \ No newline at end of file diff --git a/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj b/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj index 752184105..0184f183e 100644 --- a/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj +++ b/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj @@ -1,214 +1,218 @@ - - - - - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - Debug - ARM - - - Release - ARM - - - - 15.0 - {928899BC-F131-4343-A1AB-72F3A5787E41} - Win32Proj - 10.0.17763.0 - mrwebrtc-win32 - - - - DynamicLibrary - v141 - Unicode - - - true - - - false - true - - - - - - - - - - - - - true - - - false - - - $(MRWebRTCProjectRoot)bin\Win32\$(PlatformTarget)\$(Configuration)\ - $(MRWebRTCProjectRoot)build\mrwebrtc\Win32\$(PlatformTarget)\$(Configuration)\ - mrwebrtc - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\Win32\x86 - - - $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\Win32\x86_64 - - - - - - - Use - Level4 - true - WIN32;_WINDOWS;_USRDLL;UNICODE;_WINDLL;MR_SHARING_WIN;MRS_USE_STR_WRAPPER;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;%(PreprocessorDefinitions) - true - $(MRWebRTCProjectRoot)libs\mrwebrtc\include;$(MRWebRTCProjectRoot)libs\mrwebrtc\src;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc;$(WebRTCCoreRepoPath)webrtc\xplatform$(WebRTCCoreRepoPath)webrtc\xplatform\chromium;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\generated\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\override\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium\third_party\abseil-cpp;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\third_party\idl;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib-eventing;$(WebRTCCoreRepoPath)webrtc\xplatform\libyuv\include;%(AdditionalIncludeDirectories) - stdcpp17 - pch.h - /Zc:__cplusplus %(AdditionalOptions) - - - Windows - strmiids.lib;Msdmo.lib;dmoguids.lib;wmcodecdspuuid.lib;Secur32.lib;winmm.lib;Ole32.lib;Evr.lib;mfreadwrite.lib;mf.lib;mfuuid.lib;mfplat.lib;mfplay.lib;webrtc.lib;%(AdditionalDependencies) - $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\win\$(PlatformTarget)\$(Configuration);%(AdditionalLibraryDirectories) - DebugFull - - - - - Disabled - _DEBUG;%(PreprocessorDefinitions) - MultiThreadedDebug - - - - - MaxSpeed - true - true - NDEBUG;%(PreprocessorDefinitions) - true - MultiThreaded - - - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create - pch.h - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - - + + + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + Debug + ARM + + + Release + ARM + + + + 15.0 + {928899BC-F131-4343-A1AB-72F3A5787E41} + Win32Proj + 10.0.17763.0 + mrwebrtc-win32 + + + + DynamicLibrary + v141 + Unicode + + + true + + + false + true + + + + + + + + + + + + + true + + + false + + + $(MRWebRTCProjectRoot)bin\Win32\$(PlatformTarget)\$(Configuration)\ + $(MRWebRTCProjectRoot)build\mrwebrtc\Win32\$(PlatformTarget)\$(Configuration)\ + mrwebrtc + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\Win32\x86 + + + $(MRWebRTCProjectRoot)libs\unity\library\Runtime\Plugins\Win32\x86_64 + + + + + + + Use + Level4 + true + WIN32;_WINDOWS;_USRDLL;UNICODE;_WINDLL;MR_SHARING_WIN;MRS_USE_STR_WRAPPER;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;%(PreprocessorDefinitions) + true + $(MRWebRTCProjectRoot)libs\mrwebrtc\include;$(MRWebRTCProjectRoot)libs\mrwebrtc\src;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc;$(WebRTCCoreRepoPath)webrtc\xplatform$(WebRTCCoreRepoPath)webrtc\xplatform\chromium;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\generated\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\sdk\windows\wrapper\override\cppwinrt;$(WebRTCCoreRepoPath)webrtc\xplatform\chromium\third_party\abseil-cpp;$(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\third_party\idl;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib;$(WebRTCCoreRepoPath)webrtc\xplatform\zsLib-eventing;$(WebRTCCoreRepoPath)webrtc\xplatform\libyuv\include;%(AdditionalIncludeDirectories) + stdcpp17 + pch.h + /Zc:__cplusplus %(AdditionalOptions) + + + Windows + strmiids.lib;Msdmo.lib;dmoguids.lib;wmcodecdspuuid.lib;Secur32.lib;winmm.lib;Ole32.lib;Evr.lib;mfreadwrite.lib;mf.lib;mfuuid.lib;mfplat.lib;mfplay.lib;webrtc.lib;%(AdditionalDependencies) + $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\win\$(PlatformTarget)\$(Configuration);%(AdditionalLibraryDirectories) + DebugFull + + + + + Disabled + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebug + + + + + MaxSpeed + true + true + NDEBUG;%(PreprocessorDefinitions) + true + MultiThreaded + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Create + pch.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + \ No newline at end of file diff --git a/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj.filters b/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj.filters index 0ae632c17..b2857ea93 100644 --- a/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj.filters +++ b/tools/build/mrwebrtc/win32/mrwebrtc-win32.vcxproj.filters @@ -1,226 +1,238 @@ - - - - - src\interop - - - src\interop - - - src\interop - - - src\interop - - - src\interop - - - src\media - - - src\media - - - src\interop - - - src\interop - - - src\interop - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\interop - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src\media - - - src\media - - - src\interop - - - src - - - src\media - - - src - - - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - src\media - - - include - - - include - - - include - - - include - - - include - - - include - - - include - - - include - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src - - - src\media - - - src\media - - - src\interop - - - include - - - include - - - include - - - include - - - src\media - - - src - - - - - - - - - {4c1543fc-a25f-4a8e-91a4-3c75fd1e44bb} - - - {d0d6e8a6-6d73-4bd1-af8a-6d410addbd75} - - - {5683d251-f1a9-4d36-89cb-881b2fd36b65} - - - {2eda4eaf-b512-47aa-b587-45de3b17ef8e} - - + + + + + src\interop + + + src\interop + + + src\interop + + + src\interop + + + src\interop + + + src\media + + + src\media + + + src\interop + + + src\interop + + + src\interop + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\interop + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src\media + + + src\media + + + src\interop + + + src + + + src\media + + + src + + + src\interop + + + src\media + + + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + src\media + + + include + + + include + + + include + + + include + + + include + + + include + + + include + + + include + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src + + + src\media + + + src\media + + + src\interop + + + include + + + include + + + include + + + include + + + src\media + + + src + + + include + + + src\media + + + + + + + + + {4c1543fc-a25f-4a8e-91a4-3c75fd1e44bb} + + + {d0d6e8a6-6d73-4bd1-af8a-6d410addbd75} + + + {5683d251-f1a9-4d36-89cb-881b2fd36b65} + + + {2eda4eaf-b512-47aa-b587-45de3b17ef8e} + + \ No newline at end of file diff --git a/tools/build/mrwebrtc/win32/tests/mrwebrtc-win32-tests.vcxproj b/tools/build/mrwebrtc/win32/tests/mrwebrtc-win32-tests.vcxproj index efa6db17a..fc2a236a1 100644 --- a/tools/build/mrwebrtc/win32/tests/mrwebrtc-win32-tests.vcxproj +++ b/tools/build/mrwebrtc/win32/tests/mrwebrtc-win32-tests.vcxproj @@ -63,6 +63,7 @@ + From 8ddbf22b74d4c4ce386439cf2f6fc022ef3a6e78 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 25 May 2020 10:09:59 +0100 Subject: [PATCH 2/7] mrsOptBool fix --- .../Interop/AudioTrackSourceInterop.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs index 94339c306..ed882ddef 100644 --- a/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs +++ b/libs/Microsoft.MixedReality.WebRTC/Interop/AudioTrackSourceInterop.cs @@ -65,7 +65,7 @@ internal ref struct LocalAudioDeviceMarshalInitConfig /// public LocalAudioDeviceMarshalInitConfig(LocalAudioDeviceInitConfig settings) { - AutoGainControl = new mrsOptBool(settings?.AutoGainControl); + AutoGainControl = (mrsOptBool)settings?.AutoGainControl; } } From 9c6adeda41d59d642bc0def308e5f8678f1614a4 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 25 May 2020 12:51:40 +0100 Subject: [PATCH 3/7] fixp: audio track source frame callback --- libs/mrwebrtc/include/audio_track_source_interop.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/mrwebrtc/include/audio_track_source_interop.h b/libs/mrwebrtc/include/audio_track_source_interop.h index 77cb05c4d..2e221c400 100644 --- a/libs/mrwebrtc/include/audio_track_source_interop.h +++ b/libs/mrwebrtc/include/audio_track_source_interop.h @@ -71,6 +71,6 @@ MRS_API mrsResult MRS_CALL mrsAudioTrackSourceCreateFromDevice( MRS_API void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( mrsAudioTrackSourceHandle source_handle, mrsAudioFrameCallback callback, - void* user_data) noexcept + void* user_data) noexcept; } // extern "C" From 29bc4f8370d1d5a8555c32b332c8d0da400aeda6 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 25 May 2020 16:20:37 +0100 Subject: [PATCH 4/7] Use IReadOnlyList in public API --- .../AudioTrackSource.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs index 9a9aaedb9..248194438 100644 --- a/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs +++ b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs @@ -30,7 +30,7 @@ public class LocalAudioDeviceInitConfig /// /// The user owns the audio track source, and is in charge of keeping it alive until after all tracks using it /// are destroyed, and then dispose of it. The behavior of disposing of the track source while a track is still - /// using it is undefined. The property contains the list of tracks currently using the + /// using it is undefined. The property contains the list of tracks currently using the /// source. /// /// @@ -57,7 +57,7 @@ public string Name /// /// List of local audio tracks this source is providing raw audio frames to. /// - public List Tracks { get; private set; } = new List(); + public IReadOnlyList Tracks => _tracks; /// /// Handle to the native AudioTrackSource object. @@ -74,6 +74,11 @@ public string Name /// private string _name = string.Empty; + /// + /// Backing field for . + /// + private List _tracks = new List(); + /// /// Create an audio track source using a local audio capture device (microphone). /// @@ -107,7 +112,7 @@ public void Dispose() } // TODO - Can we support destroying the source and leaving tracks with silence instead? - if (Tracks.Count > 0) + if (_tracks.Count > 0) { throw new InvalidOperationException($"Trying to dispose of AudioTrackSource '{Name}' while still in use by one or more audio tracks."); } @@ -128,8 +133,8 @@ public void Dispose() internal void OnTrackAddedToSource(LocalAudioTrack track) { Debug.Assert(!_nativeHandle.IsClosed); - Debug.Assert(!Tracks.Contains(track)); - Tracks.Add(track); + Debug.Assert(!_tracks.Contains(track)); + _tracks.Add(track); } /// @@ -139,7 +144,7 @@ internal void OnTrackAddedToSource(LocalAudioTrack track) internal void OnTrackRemovedFromSource(LocalAudioTrack track) { Debug.Assert(!_nativeHandle.IsClosed); - bool removed = Tracks.Remove(track); + bool removed = _tracks.Remove(track); Debug.Assert(removed); } @@ -148,7 +153,7 @@ internal void OnTrackRemovedFromSource(LocalAudioTrack track) /// as a result of a peer connection owning said tracks being closed. /// /// The list of tracks not using this source anymore. - internal void OnTracksRemovedFromSource(List tracks) + internal void OnTracksRemovedFromSource(IEnumerable tracks) { Debug.Assert(!_nativeHandle.IsClosed); var remainingTracks = new List(); @@ -156,15 +161,14 @@ internal void OnTracksRemovedFromSource(List tracks) { if (track.Source == this) { - bool removed = Tracks.Remove(track); - Debug.Assert(removed); + Debug.Assert(_tracks.Contains(track)); } else { remainingTracks.Add(track); } } - Tracks = remainingTracks; + _tracks = remainingTracks; } /// From 4da3d9ca7d27446f68f1d2221fcd78cda0098d05 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 25 May 2020 16:26:08 +0100 Subject: [PATCH 5/7] Comment out no-op local audio callback --- .../include/audio_track_source_interop.h | 8 ++++---- .../src/interop/audio_track_source_interop.cpp | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libs/mrwebrtc/include/audio_track_source_interop.h b/libs/mrwebrtc/include/audio_track_source_interop.h index 2e221c400..11984fa83 100644 --- a/libs/mrwebrtc/include/audio_track_source_interop.h +++ b/libs/mrwebrtc/include/audio_track_source_interop.h @@ -68,9 +68,9 @@ MRS_API mrsResult MRS_CALL mrsAudioTrackSourceCreateFromDevice( /// not hook those callbacks, and therefore the callback will never be called. /// This is a limitation of the underlying implementation. /// See https://bugs.chromium.org/p/webrtc/issues/detail?id=11602 -MRS_API void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( - mrsAudioTrackSourceHandle source_handle, - mrsAudioFrameCallback callback, - void* user_data) noexcept; +//MRS_API void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( +// mrsAudioTrackSourceHandle source_handle, +// mrsAudioFrameCallback callback, +// void* user_data) noexcept; } // extern "C" diff --git a/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp b/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp index 24473b94c..b9473426a 100644 --- a/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp +++ b/libs/mrwebrtc/src/interop/audio_track_source_interop.cpp @@ -120,11 +120,11 @@ mrsResult MRS_CALL mrsAudioTrackSourceCreateFromDevice( return Result::kSuccess; } -void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( - mrsAudioTrackSourceHandle source_handle, - mrsAudioFrameCallback callback, - void* user_data) noexcept { - if (auto source = static_cast(source_handle)) { - source->SetCallback(AudioFrameReadyCallback{callback, user_data}); - } -} +//void MRS_CALL mrsAudioTrackSourceRegisterFrameCallback( +// mrsAudioTrackSourceHandle source_handle, +// mrsAudioFrameCallback callback, +// void* user_data) noexcept { +// if (auto source = static_cast(source_handle)) { +// source->SetCallback(AudioFrameReadyCallback{callback, user_data}); +// } +//} From 1c5ce118d71ee0e34d16d0af39f8460e017c8ad2 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Mon, 25 May 2020 16:49:48 +0100 Subject: [PATCH 6/7] Fix invalid comment reference --- libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs index 248194438..b7dbc80e5 100644 --- a/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs +++ b/libs/Microsoft.MixedReality.WebRTC/AudioTrackSource.cs @@ -30,7 +30,7 @@ public class LocalAudioDeviceInitConfig /// /// The user owns the audio track source, and is in charge of keeping it alive until after all tracks using it /// are destroyed, and then dispose of it. The behavior of disposing of the track source while a track is still - /// using it is undefined. The property contains the list of tracks currently using the + /// using it is undefined. The property contains the list of tracks currently using the /// source. /// /// From 78887f5278f91716fb75ca4fa5a17e6a0505e6d9 Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Tue, 26 May 2020 10:00:26 +0100 Subject: [PATCH 7/7] Remove copy/paste TODOs --- libs/mrwebrtc/src/media/audio_track_source.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/libs/mrwebrtc/src/media/audio_track_source.h b/libs/mrwebrtc/src/media/audio_track_source.h index 6c6abd157..e35583404 100644 --- a/libs/mrwebrtc/src/media/audio_track_source.h +++ b/libs/mrwebrtc/src/media/audio_track_source.h @@ -41,15 +41,12 @@ class AudioSourceAdapter : public webrtc::AudioSourceInterface { // // Sets the volume of the source. |volume| is in the range of [0, 10]. - // TODO(tommi): This method should be on the track and ideally volume should - // be applied in the track in a way that does not affect clones of the track. void SetVolume(double /*volume*/) override {} // Registers/unregisters observers to the audio source. void RegisterAudioObserver(AudioObserver* /*observer*/) override {} void UnregisterAudioObserver(AudioObserver* /*observer*/) override {} - // TODO(tommi): Make pure virtual. void AddSink(webrtc::AudioTrackSinkInterface* sink) override; void RemoveSink(webrtc::AudioTrackSinkInterface* sink) override;