From 7454824ea6481cae2f70ad58fb1e1eb5d3ecdc88 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Sat, 11 Sep 2021 19:27:32 +0800 Subject: [PATCH] Audio Device Optimization allow listen-only mode in AudioUnit, adjust when category changes (#2) release mic when category changes (#5) Change defaults to iOS defaults (#7) Sync audio session config (#8) feat: support bypass voice processing for iOS. (#15) Remove MacBookPro audio pan right code (#22) fix: Fix can't open mic alone when built-in AEC is enabled. (#29) feat: add audio device changes detect for windows. (#41) fix Linux compile (#47) AudioUnit: Don't rely on category switch for mic indicator to turn off (#52) Stop recording on mute (turn off mic indicator) (#55) Cherry pick audio selection from m97 release (#35) [Mac] Allow audio device selection (#21) RTCAudioDeviceModule.outputDevice / inputDevice getter and setter (#80) Allow custom audio processing by exposing AudioProcessingModule (#85) Expose audio sample buffers for Android (#89) feat: add external audio processor for android. (#103) android: make audio output attributes modifiable (#118) Fix external audio processor sample rate calculation (#108) Expose remote audio sample buffers on RTCAudioTrack (#84) Fix memory leak when creating audio CMSampleBuffer #86 Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Co-authored-by: David Zhao Co-authored-by: davidliu --- audio/audio_send_stream.cc | 5 + audio/audio_send_stream.h | 1 + audio/audio_state.cc | 64 +++- audio/audio_state.h | 5 + audio/channel_send.cc | 4 +- audio/channel_send.h | 2 + call/audio_send_stream.h | 1 + call/audio_state.h | 3 + media/engine/webrtc_voice_engine.h | 3 +- .../audio_device_data_observer.cc | 4 + modules/audio_device/audio_device_generic.h | 4 + modules/audio_device/audio_device_impl.cc | 42 ++- modules/audio_device/audio_device_impl.h | 10 +- modules/audio_device/include/audio_device.h | 23 +- modules/audio_device/mac/audio_device_mac.cc | 293 ++++++++++------- modules/audio_device/mac/audio_device_mac.h | 14 +- .../audio_device/win/audio_device_core_win.cc | 70 +++++ .../audio_device/win/audio_device_core_win.h | 33 ++ sdk/BUILD.gn | 26 ++ sdk/android/BUILD.gn | 11 + sdk/android/api/org/webrtc/AudioTrack.java | 48 +++ .../api/org/webrtc/AudioTrackSink.java | 27 ++ .../ExternalAudioProcessingFactory.java | 144 +++++++++ .../webrtc/audio/JavaAudioDeviceModule.java | 20 +- .../org/webrtc/audio/WebRtcAudioTrack.java | 23 +- sdk/android/src/jni/pc/audio_sink.cc | 39 +++ sdk/android/src/jni/pc/audio_sink.h | 41 +++ sdk/android/src/jni/pc/audio_track.cc | 26 ++ .../pc/external_audio_processing_factory.cc | 143 +++++++++ .../pc/external_audio_processing_factory.h | 68 ++++ .../pc/external_audio_processing_interface.h | 33 ++ .../src/jni/pc/external_audio_processor.cc | 72 +++++ .../src/jni/pc/external_audio_processor.h | 57 ++++ .../RTCAudioDeviceModule+Private.h | 31 ++ .../api/peerconnection/RTCAudioDeviceModule.h | 56 ++++ .../peerconnection/RTCAudioDeviceModule.mm | 294 ++++++++++++++++++ .../peerconnection/RTCAudioTrack+Private.h | 10 +- sdk/objc/api/peerconnection/RTCAudioTrack.h | 8 + sdk/objc/api/peerconnection/RTCAudioTrack.mm | 218 ++++++++++++- .../api/peerconnection/RTCIODevice+Private.h | 28 ++ sdk/objc/api/peerconnection/RTCIODevice.h | 41 +++ sdk/objc/api/peerconnection/RTCIODevice.mm | 71 +++++ .../RTCPeerConnectionFactory+Native.h | 6 +- .../peerconnection/RTCPeerConnectionFactory.h | 13 + .../RTCPeerConnectionFactory.mm | 83 ++++- .../RTCPeerConnectionFactoryBuilder.mm | 3 +- sdk/objc/base/RTCAudioRenderer.h | 34 ++ .../components/audio/RTCAudioBuffer+Private.h | 29 ++ sdk/objc/components/audio/RTCAudioBuffer.h | 38 +++ sdk/objc/components/audio/RTCAudioBuffer.mm | 55 ++++ .../RTCAudioCustomProcessingAdapter+Private.h | 43 +++ .../audio/RTCAudioCustomProcessingAdapter.h | 28 ++ .../audio/RTCAudioCustomProcessingAdapter.mm | 139 +++++++++ .../audio/RTCAudioCustomProcessingDelegate.h | 52 ++++ .../audio/RTCAudioProcessingConfig+Private.h | 29 ++ .../audio/RTCAudioProcessingConfig.h | 31 ++ .../audio/RTCAudioProcessingConfig.mm | 51 +++ .../audio/RTCAudioProcessingModule.h | 33 ++ .../audio/RTCAudioSession+Configuration.mm | 27 ++ sdk/objc/components/audio/RTCAudioSession.mm | 3 +- .../audio/RTCAudioSessionConfiguration.m | 8 +- .../RTCDefaultAudioProcessingModule+Private.h | 29 ++ .../audio/RTCDefaultAudioProcessingModule.h | 47 +++ .../audio/RTCDefaultAudioProcessingModule.mm | 96 ++++++ sdk/objc/native/api/audio_device_module.mm | 4 + sdk/objc/native/src/audio/audio_device_ios.h | 12 +- sdk/objc/native/src/audio/audio_device_ios.mm | 113 +++++-- .../src/audio/audio_device_module_ios.h | 3 + .../src/audio/audio_device_module_ios.mm | 6 + .../src/audio/voice_processing_audio_unit.h | 2 +- .../src/audio/voice_processing_audio_unit.mm | 28 +- 71 files changed, 2946 insertions(+), 215 deletions(-) create mode 100644 sdk/android/api/org/webrtc/AudioTrackSink.java create mode 100644 sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java create mode 100644 sdk/android/src/jni/pc/audio_sink.cc create mode 100644 sdk/android/src/jni/pc/audio_sink.h create mode 100644 sdk/android/src/jni/pc/external_audio_processing_factory.cc create mode 100644 sdk/android/src/jni/pc/external_audio_processing_factory.h create mode 100644 sdk/android/src/jni/pc/external_audio_processing_interface.h create mode 100644 sdk/android/src/jni/pc/external_audio_processor.cc create mode 100644 sdk/android/src/jni/pc/external_audio_processor.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule.h create mode 100644 sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm create mode 100644 sdk/objc/api/peerconnection/RTCIODevice+Private.h create mode 100644 sdk/objc/api/peerconnection/RTCIODevice.h create mode 100644 sdk/objc/api/peerconnection/RTCIODevice.mm create mode 100644 sdk/objc/base/RTCAudioRenderer.h create mode 100644 sdk/objc/components/audio/RTCAudioBuffer+Private.h create mode 100644 sdk/objc/components/audio/RTCAudioBuffer.h create mode 100644 sdk/objc/components/audio/RTCAudioBuffer.mm create mode 100644 sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h create mode 100644 sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h create mode 100644 sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.mm create mode 100644 sdk/objc/components/audio/RTCAudioCustomProcessingDelegate.h create mode 100644 sdk/objc/components/audio/RTCAudioProcessingConfig+Private.h create mode 100644 sdk/objc/components/audio/RTCAudioProcessingConfig.h create mode 100644 sdk/objc/components/audio/RTCAudioProcessingConfig.mm create mode 100644 sdk/objc/components/audio/RTCAudioProcessingModule.h create mode 100644 sdk/objc/components/audio/RTCDefaultAudioProcessingModule+Private.h create mode 100644 sdk/objc/components/audio/RTCDefaultAudioProcessingModule.h create mode 100644 sdk/objc/components/audio/RTCDefaultAudioProcessingModule.mm diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 59b0ea5b5e..0804bc8286 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -415,6 +415,11 @@ void AudioSendStream::SetMuted(bool muted) { channel_send_->SetInputMute(muted); } +bool AudioSendStream::GetMuted() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return channel_send_->InputMute(); +} + webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { return GetStats(true); } diff --git a/audio/audio_send_stream.h b/audio/audio_send_stream.h index a37c8fd452..b7c265fa30 100644 --- a/audio/audio_send_stream.h +++ b/audio/audio_send_stream.h @@ -94,6 +94,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, int payload_frequency, int event, int duration_ms) override; + bool GetMuted() override; void SetMuted(bool muted) override; webrtc::AudioSendStream::Stats GetStats() const override; webrtc::AudioSendStream::Stats GetStats( diff --git a/audio/audio_state.cc b/audio/audio_state.cc index 6f20e7b128..3a21d9b3a9 100644 --- a/audio/audio_state.cc +++ b/audio/audio_state.cc @@ -98,14 +98,26 @@ void AudioState::AddSendingStream(webrtc::AudioSendStream* stream, UpdateAudioTransportWithSendingStreams(); // Make sure recording is initialized; start recording if enabled. - auto* adm = config_.audio_device_module.get(); - if (!adm->Recording()) { - if (adm->InitRecording() == 0) { - if (recording_enabled_) { - adm->StartRecording(); + if (ShouldRecord()) { + auto* adm = config_.audio_device_module.get(); + if (!adm->Recording()) { + if (adm->InitRecording() == 0) { + if (recording_enabled_) { + + // TODO: Verify if the following windows only logic is still required. +#if defined(WEBRTC_WIN) + if (adm->BuiltInAECIsAvailable() && !adm->Playing()) { + if (!adm->PlayoutIsInitialized()) { + adm->InitPlayout(); + } + adm->StartPlayout(); + } +#endif + adm->StartRecording(); + } + } else { + RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording."; } - } else { - RTC_DLOG_F(LS_ERROR) << "Failed to initialize recording."; } } } @@ -115,7 +127,8 @@ void AudioState::RemoveSendingStream(webrtc::AudioSendStream* stream) { auto count = sending_streams_.erase(stream); RTC_DCHECK_EQ(1, count); UpdateAudioTransportWithSendingStreams(); - if (sending_streams_.empty()) { + + if (!ShouldRecord()) { config_.audio_device_module->StopRecording(); } } @@ -143,7 +156,7 @@ void AudioState::SetRecording(bool enabled) { if (recording_enabled_ != enabled) { recording_enabled_ = enabled; if (enabled) { - if (!sending_streams_.empty()) { + if (ShouldRecord()) { config_.audio_device_module->StartRecording(); } } else { @@ -203,6 +216,39 @@ void AudioState::UpdateNullAudioPollerState() { null_audio_poller_.Stop(); } } + +void AudioState::OnMuteStreamChanged() { + + auto* adm = config_.audio_device_module.get(); + bool should_record = ShouldRecord(); + + if (should_record && !adm->Recording()) { + if (adm->InitRecording() == 0) { + adm->StartRecording(); + } + } else if (!should_record && adm->Recording()) { + adm->StopRecording(); + } +} + +bool AudioState::ShouldRecord() { + // no streams to send + if (sending_streams_.empty()) { + return false; + } + + int stream_count = sending_streams_.size(); + + int muted_count = 0; + for (const auto& kv : sending_streams_) { + if (kv.first->GetMuted()) { + muted_count++; + } + } + + return muted_count != stream_count; +} + } // namespace internal rtc::scoped_refptr AudioState::Create( diff --git a/audio/audio_state.h b/audio/audio_state.h index 88aaaa3697..f21cca771e 100644 --- a/audio/audio_state.h +++ b/audio/audio_state.h @@ -47,6 +47,8 @@ class AudioState : public webrtc::AudioState { void SetStereoChannelSwapping(bool enable) override; + void OnMuteStreamChanged() override; + AudioDeviceModule* audio_device_module() { RTC_DCHECK(config_.audio_device_module); return config_.audio_device_module.get(); @@ -64,6 +66,9 @@ class AudioState : public webrtc::AudioState { void UpdateAudioTransportWithSendingStreams(); void UpdateNullAudioPollerState() RTC_RUN_ON(&thread_checker_); + // Returns true when at least 1 stream exists and all streams are not muted. + bool ShouldRecord(); + SequenceChecker thread_checker_; SequenceChecker process_thread_checker_{SequenceChecker::kDetached}; const webrtc::AudioState::Config config_; diff --git a/audio/channel_send.cc b/audio/channel_send.cc index 4a2700177b..bc474ee33d 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc @@ -100,6 +100,8 @@ class ChannelSend : public ChannelSendInterface, // Muting, Volume and Level. void SetInputMute(bool enable) override; + bool InputMute() const override; + // Stats. ANAStats GetANAStatistics() const override; @@ -163,8 +165,6 @@ class ChannelSend : public ChannelSendInterface, size_t payloadSize, int64_t absolute_capture_timestamp_ms) override; - bool InputMute() const; - int32_t SendRtpAudio(AudioFrameType frameType, uint8_t payloadType, uint32_t rtp_timestamp_without_offset, diff --git a/audio/channel_send.h b/audio/channel_send.h index b6a6a37bf5..c16be987b4 100644 --- a/audio/channel_send.h +++ b/audio/channel_send.h @@ -83,6 +83,8 @@ class ChannelSendInterface { virtual bool SendTelephoneEventOutband(int event, int duration_ms) = 0; virtual void OnBitrateAllocation(BitrateAllocationUpdate update) = 0; virtual int GetTargetBitrate() const = 0; + + virtual bool InputMute() const = 0; virtual void SetInputMute(bool muted) = 0; virtual void ProcessAndEncodeAudio( diff --git a/call/audio_send_stream.h b/call/audio_send_stream.h index 9c2fad652f..7e73ab2ce6 100644 --- a/call/audio_send_stream.h +++ b/call/audio_send_stream.h @@ -190,6 +190,7 @@ class AudioSendStream : public AudioSender { int event, int duration_ms) = 0; + virtual bool GetMuted() = 0; virtual void SetMuted(bool muted) = 0; virtual Stats GetStats() const = 0; diff --git a/call/audio_state.h b/call/audio_state.h index 79fb5cf981..85f04758dd 100644 --- a/call/audio_state.h +++ b/call/audio_state.h @@ -59,6 +59,9 @@ class AudioState : public rtc::RefCountInterface { virtual void SetStereoChannelSwapping(bool enable) = 0; + // Notify the AudioState that a stream updated it's mute state. + virtual void OnMuteStreamChanged() = 0; + static rtc::scoped_refptr Create( const AudioState::Config& config); diff --git a/media/engine/webrtc_voice_engine.h b/media/engine/webrtc_voice_engine.h index b28b9652bb..9d18ef6302 100644 --- a/media/engine/webrtc_voice_engine.h +++ b/media/engine/webrtc_voice_engine.h @@ -132,6 +132,8 @@ class WebRtcVoiceEngine final : public VoiceEngineInterface { absl::optional GetAudioDeviceStats() override; + // Moved to public so WebRtcVoiceMediaChannel can access it. + webrtc::AudioState* audio_state(); private: // Every option that is "set" will be applied. Every option not "set" will be @@ -145,7 +147,6 @@ class WebRtcVoiceEngine final : public VoiceEngineInterface { webrtc::AudioDeviceModule* adm(); webrtc::AudioProcessing* apm() const; - webrtc::AudioState* audio_state(); std::vector CollectCodecs( const std::vector& specs) const; diff --git a/modules/audio_device/audio_device_data_observer.cc b/modules/audio_device/audio_device_data_observer.cc index 0524830327..88a8301c4f 100644 --- a/modules/audio_device/audio_device_data_observer.cc +++ b/modules/audio_device/audio_device_data_observer.cc @@ -307,6 +307,10 @@ class ADMWrapper : public AudioDeviceModule, public AudioTransport { } #endif // WEBRTC_IOS + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override { + return impl_->SetAudioDeviceSink(sink); + } + protected: rtc::scoped_refptr impl_; AudioDeviceDataObserver* legacy_observer_ = nullptr; diff --git a/modules/audio_device/audio_device_generic.h b/modules/audio_device/audio_device_generic.h index 41e24eb3b0..0585129de4 100644 --- a/modules/audio_device/audio_device_generic.h +++ b/modules/audio_device/audio_device_generic.h @@ -135,6 +135,10 @@ class AudioDeviceGeneric { virtual int GetRecordAudioParameters(AudioParameters* params) const; #endif // WEBRTC_IOS + virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink) { return -1; } + virtual int32_t GetPlayoutDevice() const { return -1; } + virtual int32_t GetRecordingDevice() const { return -1; } + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) = 0; virtual ~AudioDeviceGeneric() {} diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index 80ed928933..0cd86d7cdb 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -63,15 +63,17 @@ namespace webrtc { rtc::scoped_refptr AudioDeviceModule::Create( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) { RTC_DLOG(LS_INFO) << __FUNCTION__; - return AudioDeviceModule::CreateForTest(audio_layer, task_queue_factory); + return AudioDeviceModule::CreateForTest(audio_layer, task_queue_factory, bypass_voice_processing); } // static rtc::scoped_refptr AudioDeviceModule::CreateForTest( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) { RTC_DLOG(LS_INFO) << __FUNCTION__; // The "AudioDeviceModule::kWindowsCoreAudio2" audio layer has its own @@ -92,7 +94,7 @@ rtc::scoped_refptr AudioDeviceModule::CreateForTest( // Create the generic reference counted (platform independent) implementation. auto audio_device = rtc::make_ref_counted( - audio_layer, task_queue_factory); + audio_layer, task_queue_factory, bypass_voice_processing); // Ensure that the current platform is supported. if (audio_device->CheckPlatform() == -1) { @@ -115,8 +117,13 @@ rtc::scoped_refptr AudioDeviceModule::CreateForTest( AudioDeviceModuleImpl::AudioDeviceModuleImpl( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory) - : audio_layer_(audio_layer), audio_device_buffer_(task_queue_factory) { + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing) + : audio_layer_(audio_layer), +#if defined(WEBRTC_IOS) + bypass_voice_processing_(bypass_voice_processing), +#endif + audio_device_buffer_(task_queue_factory) { RTC_DLOG(LS_INFO) << __FUNCTION__; } @@ -240,7 +247,7 @@ int32_t AudioDeviceModuleImpl::CreatePlatformSpecificObjects() { #if defined(WEBRTC_IOS) if (audio_layer == kPlatformDefaultAudio) { audio_device_.reset( - new ios_adm::AudioDeviceIOS(/*bypass_voice_processing=*/false)); + new ios_adm::AudioDeviceIOS(/*bypass_voice_processing=*/bypass_voice_processing_)); RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized."; } // END #if defined(WEBRTC_IOS) @@ -895,6 +902,27 @@ int AudioDeviceModuleImpl::GetRecordAudioParameters( } #endif // WEBRTC_IOS +int32_t AudioDeviceModuleImpl::SetAudioDeviceSink(AudioDeviceSink* sink) const { + RTC_LOG(LS_INFO) << __FUNCTION__ << "(" << sink << ")"; + int32_t ok = audio_device_->SetAudioDeviceSink(sink); + RTC_LOG(LS_INFO) << "output: " << ok; + return ok; +} + +int32_t AudioDeviceModuleImpl::GetPlayoutDevice() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + int32_t r = audio_device_->GetPlayoutDevice(); + RTC_LOG(LS_INFO) << "output: " << r; + return r; +} + +int32_t AudioDeviceModuleImpl::GetRecordingDevice() const { + RTC_LOG(LS_INFO) << __FUNCTION__; + int32_t r = audio_device_->GetRecordingDevice(); + RTC_LOG(LS_INFO) << "output: " << r; + return r; +} + AudioDeviceModuleImpl::PlatformType AudioDeviceModuleImpl::Platform() const { RTC_LOG(LS_INFO) << __FUNCTION__; return platform_type_; diff --git a/modules/audio_device/audio_device_impl.h b/modules/audio_device/audio_device_impl.h index 46d91a46c8..fd9b62c65b 100644 --- a/modules/audio_device/audio_device_impl.h +++ b/modules/audio_device/audio_device_impl.h @@ -47,7 +47,8 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { int32_t AttachAudioBuffer(); AudioDeviceModuleImpl(AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); // If `create_detached` is true, created ADM can be used on another thread // compared to the one on which it was created. It's useful for testing. AudioDeviceModuleImpl(AudioLayer audio_layer, @@ -155,6 +156,10 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { int GetRecordAudioParameters(AudioParameters* params) const override; #endif // WEBRTC_IOS + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override; + int32_t GetPlayoutDevice() const override; + int32_t GetRecordingDevice() const override; + AudioDeviceBuffer* GetAudioDeviceBuffer() { return &audio_device_buffer_; } int RestartPlayoutInternally() override { return -1; } @@ -169,6 +174,9 @@ class AudioDeviceModuleImpl : public AudioDeviceModuleForTest { AudioLayer audio_layer_; PlatformType platform_type_ = kPlatformNotSupported; bool initialized_ = false; +#if defined(WEBRTC_IOS) + bool bypass_voice_processing_; +#endif AudioDeviceBuffer audio_device_buffer_; std::unique_ptr audio_device_; }; diff --git a/modules/audio_device/include/audio_device.h b/modules/audio_device/include/audio_device.h index 47d2aecfa7..58019cc24f 100644 --- a/modules/audio_device/include/audio_device.h +++ b/modules/audio_device/include/audio_device.h @@ -12,16 +12,25 @@ #define MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_H_ #include "absl/types/optional.h" -#include "api/ref_count.h" #include "api/scoped_refptr.h" #include "api/task_queue/task_queue_factory.h" #include "modules/audio_device/include/audio_device_defines.h" +#include "rtc_base/ref_count.h" namespace webrtc { class AudioDeviceModuleForTest; -class AudioDeviceModule : public webrtc::RefCountInterface { +// Sink for callbacks related to a audio device. +class AudioDeviceSink { + public: + virtual ~AudioDeviceSink() = default; + + // input/output devices updated or default device changed + virtual void OnDevicesUpdated() = 0; +}; + +class AudioDeviceModule : public rtc::RefCountInterface { public: enum AudioLayer { kPlatformDefaultAudio = 0, @@ -56,12 +65,14 @@ class AudioDeviceModule : public webrtc::RefCountInterface { // Creates a default ADM for usage in production code. static rtc::scoped_refptr Create( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); // Creates an ADM with support for extra test methods. Don't use this factory // in production code. static rtc::scoped_refptr CreateForTest( AudioLayer audio_layer, - TaskQueueFactory* task_queue_factory); + TaskQueueFactory* task_queue_factory, + bool bypass_voice_processing = false); // Retrieve the currently utilized audio layer virtual int32_t ActiveAudioLayer(AudioLayer* audioLayer) const = 0; @@ -171,6 +182,10 @@ class AudioDeviceModule : public webrtc::RefCountInterface { virtual int GetRecordAudioParameters(AudioParameters* params) const = 0; #endif // WEBRTC_IOS + virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const { return -1; } + virtual int32_t GetPlayoutDevice() const { return -1; } + virtual int32_t GetRecordingDevice() const { return -1; } + protected: ~AudioDeviceModule() override {} }; diff --git a/modules/audio_device/mac/audio_device_mac.cc b/modules/audio_device/mac/audio_device_mac.cc index ed7b0e4669..0a371737b3 100644 --- a/modules/audio_device/mac/audio_device_mac.cc +++ b/modules/audio_device/mac/audio_device_mac.cc @@ -119,8 +119,6 @@ AudioDeviceMac::AudioDeviceMac() _twoDevices(true), _doStop(false), _doStopRec(false), - _macBookPro(false), - _macBookProPanRight(false), _captureLatencyUs(0), _renderLatencyUs(0), _captureDelayUs(0), @@ -277,8 +275,11 @@ AudioDeviceGeneric::InitStatus AudioDeviceMac::Init() { // but now must be explicitly specified. HAL would otherwise try to use the // main thread to issue notifications. AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyRunLoop, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster + }; + CFRunLoopRef runLoop = NULL; UInt32 size = sizeof(CFRunLoopRef); int aoerr = AudioObjectSetPropertyData( @@ -294,22 +295,15 @@ AudioDeviceGeneric::InitStatus AudioDeviceMac::Init() { WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); - // Determine if this is a MacBook Pro - _macBookPro = false; - _macBookProPanRight = false; - char buf[128]; - size_t length = sizeof(buf); - memset(buf, 0, length); + // Listen for default output device change. + propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; + WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); - int intErr = sysctlbyname("hw.model", buf, &length, NULL, 0); - if (intErr != 0) { - RTC_LOG(LS_ERROR) << "Error in sysctlbyname(): " << err; - } else { - RTC_LOG(LS_VERBOSE) << "Hardware model: " << buf; - if (strncmp(buf, "MacBookPro", 10) == 0) { - _macBookPro = true; - } - } + // Listen for default input device change. + propertyAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice; + WEBRTC_CA_LOG_ERR(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); _initialized = true; @@ -337,9 +331,21 @@ int32_t AudioDeviceMac::Terminate() { OSStatus err = noErr; int retVal = 0; + // Remove listeners for global scope. AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyDevices, // selector + kAudioObjectPropertyScopeGlobal, // scope + kAudioObjectPropertyElementMaster // element + }; + + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); + + propertyAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice; WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( kAudioObjectSystemObject, &propertyAddress, &objectListenerProc, this)); @@ -789,6 +795,14 @@ int16_t AudioDeviceMac::PlayoutDevices() { MaxNumberDevices); } +int32_t AudioDeviceMac::GetPlayoutDevice() const { + if (_outputDeviceIsSpecified) { + return _outputDeviceIndex; + } + + return 0; +} + int32_t AudioDeviceMac::SetPlayoutDevice(uint16_t index) { MutexLock lock(&mutex_); @@ -830,13 +844,11 @@ int32_t AudioDeviceMac::PlayoutDeviceName(uint16_t index, } memset(name, 0, kAdmMaxDeviceNameSize); - - if (guid != NULL) { - memset(guid, 0, kAdmMaxGuidSize); - } + memset(guid, 0, kAdmMaxGuidSize); return GetDeviceName(kAudioDevicePropertyScopeOutput, index, - rtc::ArrayView(name, kAdmMaxDeviceNameSize)); + rtc::ArrayView(name, kAdmMaxDeviceNameSize), + rtc::ArrayView(guid, kAdmMaxGuidSize)); } int32_t AudioDeviceMac::RecordingDeviceName(uint16_t index, @@ -855,7 +867,8 @@ int32_t AudioDeviceMac::RecordingDeviceName(uint16_t index, } return GetDeviceName(kAudioDevicePropertyScopeInput, index, - rtc::ArrayView(name, kAdmMaxDeviceNameSize)); + rtc::ArrayView(name, kAdmMaxDeviceNameSize), + rtc::ArrayView(guid, kAdmMaxGuidSize)); } int16_t AudioDeviceMac::RecordingDevices() { @@ -864,6 +877,14 @@ int16_t AudioDeviceMac::RecordingDevices() { MaxNumberDevices); } +int32_t AudioDeviceMac::GetRecordingDevice() const { + if (_inputDeviceIsSpecified) { + return _inputDeviceIndex; + } + + return 0; +} + int32_t AudioDeviceMac::SetRecordingDevice(uint16_t index) { if (_recIsInitialized) { return -1; @@ -979,34 +1000,8 @@ int32_t AudioDeviceMac::InitPlayout() { _renderDeviceIsAlive = 1; _doStop = false; - // The internal microphone of a MacBook Pro is located under the left speaker - // grille. When the internal speakers are in use, we want to fully stereo - // pan to the right. AudioObjectPropertyAddress propertyAddress = { kAudioDevicePropertyDataSource, kAudioDevicePropertyScopeOutput, 0}; - if (_macBookPro) { - _macBookProPanRight = false; - Boolean hasProperty = - AudioObjectHasProperty(_outputDeviceID, &propertyAddress); - if (hasProperty) { - UInt32 dataSource = 0; - size = sizeof(dataSource); - WEBRTC_CA_LOG_WARN(AudioObjectGetPropertyData( - _outputDeviceID, &propertyAddress, 0, NULL, &size, &dataSource)); - - if (dataSource == 'ispk') { - _macBookProPanRight = true; - RTC_LOG(LS_VERBOSE) - << "MacBook Pro using internal speakers; stereo panning right"; - } else { - RTC_LOG(LS_VERBOSE) << "MacBook Pro not using internal speakers"; - } - - // Add a listener to determine if the status changes. - WEBRTC_CA_LOG_WARN(AudioObjectAddPropertyListener( - _outputDeviceID, &propertyAddress, &objectListenerProc, this)); - } - } // Get current stream description propertyAddress.mSelector = kAudioDevicePropertyStreamFormat; @@ -1297,7 +1292,7 @@ int32_t AudioDeviceMac::StartRecording() { while (CaptureWorkerThread()) { } }, - "CaptureWorkerThread", + "Audio_CaptureWorkerThread", rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime)); OSStatus err = noErr; @@ -1390,7 +1385,11 @@ int32_t AudioDeviceMac::StopRecording() { // Remove listeners. AudioObjectPropertyAddress propertyAddress = { - kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeInput, 0}; + kAudioDevicePropertyStreamFormat, // selector + kAudioDevicePropertyScopeInput, // scope + 0, // element + }; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( _inputDeviceID, &propertyAddress, &objectListenerProc, this)); @@ -1434,7 +1433,7 @@ int32_t AudioDeviceMac::StartPlayout() { while (RenderWorkerThread()) { } }, - "RenderWorkerThread", + "Audio_RenderWorkerThread", rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime)); if (_twoDevices || !_recording) { @@ -1503,7 +1502,11 @@ int32_t AudioDeviceMac::StopPlayout() { // Remove listeners. AudioObjectPropertyAddress propertyAddress = { - kAudioDevicePropertyStreamFormat, kAudioDevicePropertyScopeOutput, 0}; + kAudioDevicePropertyStreamFormat, // selector + kAudioDevicePropertyScopeOutput, // scope + 0, // element + }; + WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( _outputDeviceID, &propertyAddress, &objectListenerProc, this)); @@ -1511,16 +1514,6 @@ int32_t AudioDeviceMac::StopPlayout() { WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( _outputDeviceID, &propertyAddress, &objectListenerProc, this)); - if (_macBookPro) { - Boolean hasProperty = - AudioObjectHasProperty(_outputDeviceID, &propertyAddress); - if (hasProperty) { - propertyAddress.mSelector = kAudioDevicePropertyDataSource; - WEBRTC_CA_LOG_WARN(AudioObjectRemovePropertyListener( - _outputDeviceID, &propertyAddress, &objectListenerProc, this)); - } - } - _playIsInitialized = false; _playing = false; @@ -1548,8 +1541,11 @@ int32_t AudioDeviceMac::GetNumberDevices(const AudioObjectPropertyScope scope, OSStatus err = noErr; AudioObjectPropertyAddress propertyAddress = { - kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, - kAudioObjectPropertyElementMaster}; + kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster, + }; + UInt32 size = 0; WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyDataSize( kAudioObjectSystemObject, &propertyAddress, 0, NULL, &size)); @@ -1648,7 +1644,8 @@ int32_t AudioDeviceMac::GetNumberDevices(const AudioObjectPropertyScope scope, int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, const uint16_t index, - rtc::ArrayView name) { + rtc::ArrayView name, + rtc::ArrayView guid) { OSStatus err = noErr; AudioDeviceID deviceIds[MaxNumberDevices]; @@ -1685,10 +1682,9 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, isDefaultDevice = true; } } - AudioObjectPropertyAddress propertyAddress = {kAudioDevicePropertyDeviceName, scope, 0}; - + rtc::SimpleStringBuilder guid_ss(guid); if (isDefaultDevice) { std::array devName; UInt32 len = devName.size(); @@ -1698,6 +1694,7 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, rtc::SimpleStringBuilder ss(name); ss.AppendFormat("default (%s)", devName.data()); + guid_ss << "default"; } else { if (index < numberDevices) { usedID = deviceIds[index]; @@ -1705,7 +1702,7 @@ int32_t AudioDeviceMac::GetDeviceName(const AudioObjectPropertyScope scope, usedID = index; } UInt32 len = name.size(); - + guid_ss << std::to_string(deviceIds[index]); WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( usedID, &propertyAddress, 0, NULL, &len, name.data())); } @@ -1916,6 +1913,66 @@ OSStatus AudioDeviceMac::implObjectListenerProc( HandleDataSourceChange(objectId, addresses[i]); } else if (addresses[i].mSelector == kAudioDeviceProcessorOverload) { HandleProcessorOverload(addresses[i]); + } else if (addresses[i].mSelector == kAudioHardwarePropertyDefaultOutputDevice) { + RTC_LOG(LS_VERBOSE) << "kAudioHardwarePropertyDefaultOutputDevice"; + // default audio output device changed + HandleDefaultOutputDeviceChange(); + } else if (addresses[i].mSelector == kAudioHardwarePropertyDefaultInputDevice) { + RTC_LOG(LS_VERBOSE) << "kAudioHardwarePropertyDefaultInputDevice"; + // default audio input device changed + HandleDefaultInputDeviceChange(); + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleDefaultOutputDeviceChange() { + + if (SpeakerIsInitialized()) { + RTC_LOG(LS_WARNING) << "Default audio output device has changed"; + int32_t renderDeviceIsAlive = _renderDeviceIsAlive; + bool wasPlaying = _playing && renderDeviceIsAlive == 1; + + if (wasPlaying && _outputDeviceIsSpecified && _outputDeviceIndex == 0) { + + StopPlayout(); + + // default is already selected _outputDeviceIndex(0) + // re-init and start playout + InitPlayout(); + StartPlayout(); + } + + // Notify default output device updated + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } + } + + return 0; +} + +int32_t AudioDeviceMac::HandleDefaultInputDeviceChange() { + + if (MicrophoneIsInitialized()) { + RTC_LOG(LS_WARNING) << "Default audio input device has changed"; + int32_t captureDeviceIsAlive = _captureDeviceIsAlive; + bool wasRecording = _recording && captureDeviceIsAlive == 1; + + if (wasRecording && _inputDeviceIsSpecified && _inputDeviceIndex == 0) { + + StopRecording(); + + // default is already selected _inputDeviceIndex(0) + // re-init and start recording + InitRecording(); + StartRecording(); + } + + // Notify default input device updated + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); } } @@ -1938,9 +1995,29 @@ int32_t AudioDeviceMac::HandleDeviceChange() { &size, &deviceIsAlive); if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { - RTC_LOG(LS_WARNING) << "Capture device is not alive (probably removed)"; - _captureDeviceIsAlive = 0; - _mixerManager.CloseMicrophone(); + RTC_LOG(LS_WARNING) << "Audio input device is not alive (probably removed) deviceID: " << _inputDeviceID; + //AtomicSet32(&_captureDeviceIsAlive, 0); + + // Logic to switch to default device (if exists) + // when the current device is not alive anymore + int32_t captureDeviceIsAlive = _captureDeviceIsAlive; + bool wasRecording = _recording && captureDeviceIsAlive == 1; + + StopRecording(); + + // was playing & default device exists + if (wasRecording && SetRecordingDevice(0) == 0) { + InitRecording(); + StartRecording(); + } else { + _mixerManager.CloseMicrophone(); + } + + // Notify input device removed + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); @@ -1957,9 +2034,29 @@ int32_t AudioDeviceMac::HandleDeviceChange() { &size, &deviceIsAlive); if (err == kAudioHardwareBadDeviceError || deviceIsAlive == 0) { - RTC_LOG(LS_WARNING) << "Render device is not alive (probably removed)"; - _renderDeviceIsAlive = 0; - _mixerManager.CloseSpeaker(); + RTC_LOG(LS_WARNING) << "Audio output device is not alive (probably removed) deviceID: " << _outputDeviceID; + // AtomicSet32(&_renderDeviceIsAlive, 0); // StopPlayout() does this + + // Logic to switch to default device (if exists) + // when the current device is not alive anymore + int32_t renderDeviceIsAlive = _renderDeviceIsAlive; + bool wasPlaying = _playing && renderDeviceIsAlive == 1; + + StopPlayout(); + + // was playing & default device exists + if (wasPlaying && SetPlayoutDevice(0) == 0) { + InitPlayout(); + StartPlayout(); + } else { + _mixerManager.CloseSpeaker(); + } + + // Notify output device removed + if (audio_device_module_sink_) { + audio_device_module_sink_->OnDevicesUpdated(); + } + } else if (err != noErr) { logCAMsg(rtc::LS_ERROR, "Error in AudioDeviceGetProperty()", (const char*)&err); @@ -2061,28 +2158,10 @@ int32_t AudioDeviceMac::HandleStreamFormatChange( int32_t AudioDeviceMac::HandleDataSourceChange( const AudioObjectID objectId, const AudioObjectPropertyAddress propertyAddress) { - OSStatus err = noErr; - - if (_macBookPro && - propertyAddress.mScope == kAudioDevicePropertyScopeOutput) { - RTC_LOG(LS_VERBOSE) << "Data source changed"; - - _macBookProPanRight = false; - UInt32 dataSource = 0; - UInt32 size = sizeof(UInt32); - WEBRTC_CA_RETURN_ON_ERR(AudioObjectGetPropertyData( - objectId, &propertyAddress, 0, NULL, &size, &dataSource)); - if (dataSource == 'ispk') { - _macBookProPanRight = true; - RTC_LOG(LS_VERBOSE) - << "MacBook Pro using internal speakers; stereo panning right"; - } else { - RTC_LOG(LS_VERBOSE) << "MacBook Pro not using internal speakers"; - } - } return 0; } + int32_t AudioDeviceMac::HandleProcessorOverload( const AudioObjectPropertyAddress propertyAddress) { // TODO(xians): we probably want to notify the user in some way of the @@ -2400,24 +2479,6 @@ bool AudioDeviceMac::RenderWorkerThread() { uint32_t nOutSamples = nSamples * _outDesiredFormat.mChannelsPerFrame; SInt16* pPlayBuffer = (SInt16*)&playBuffer; - if (_macBookProPanRight && (_playChannels == 2)) { - // Mix entirely into the right channel and zero the left channel. - SInt32 sampleInt32 = 0; - for (uint32_t sampleIdx = 0; sampleIdx < nOutSamples; sampleIdx += 2) { - sampleInt32 = pPlayBuffer[sampleIdx]; - sampleInt32 += pPlayBuffer[sampleIdx + 1]; - sampleInt32 /= 2; - - if (sampleInt32 > 32767) { - sampleInt32 = 32767; - } else if (sampleInt32 < -32768) { - sampleInt32 = -32768; - } - - pPlayBuffer[sampleIdx] = 0; - pPlayBuffer[sampleIdx + 1] = static_cast(sampleInt32); - } - } PaUtil_WriteRingBuffer(_paRenderBuffer, pPlayBuffer, nOutSamples); diff --git a/modules/audio_device/mac/audio_device_mac.h b/modules/audio_device/mac/audio_device_mac.h index bb06395d03..6cb5482a84 100644 --- a/modules/audio_device/mac/audio_device_mac.h +++ b/modules/audio_device/mac/audio_device_mac.h @@ -154,6 +154,13 @@ class AudioDeviceMac : public AudioDeviceGeneric { virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) RTC_LOCKS_EXCLUDED(mutex_); + virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink) RTC_LOCKS_EXCLUDED(mutex_) { + audio_device_module_sink_ = sink; + return 0; + } + virtual int32_t GetPlayoutDevice() const; + virtual int32_t GetRecordingDevice() const; + private: int32_t InitSpeakerLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); int32_t InitMicrophoneLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); @@ -180,7 +187,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { int32_t GetDeviceName(AudioObjectPropertyScope scope, uint16_t index, - rtc::ArrayView name); + rtc::ArrayView name, + rtc::ArrayView guid); int32_t InitDevice(uint16_t userDeviceIndex, AudioDeviceID& deviceId, @@ -201,6 +209,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { const AudioObjectPropertyAddress addresses[]); int32_t HandleDeviceChange(); + int32_t HandleDefaultOutputDeviceChange(); + int32_t HandleDefaultInputDeviceChange(); int32_t HandleStreamFormatChange(AudioObjectID objectId, AudioObjectPropertyAddress propertyAddress); @@ -343,6 +353,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { // Typing detection // 0x5c is key "9", after that comes function keys. bool prev_key_state_[0x5d]; + + AudioDeviceSink *audio_device_module_sink_ = nullptr; }; } // namespace webrtc diff --git a/modules/audio_device/win/audio_device_core_win.cc b/modules/audio_device/win/audio_device_core_win.cc index aa8b6a9ebe..c1c2b32a9b 100644 --- a/modules/audio_device/win/audio_device_core_win.cc +++ b/modules/audio_device/win/audio_device_core_win.cc @@ -461,6 +461,8 @@ AudioDeviceWindowsCore::AudioDeviceWindowsCore() _playChannelsPrioList[0] = 2; // stereo is prio 1 _playChannelsPrioList[1] = 1; // mono is prio 2 + _deviceStateListener = new DeviceStateListener(); + HRESULT hr; // We know that this API will work since it has already been verified in @@ -474,6 +476,8 @@ AudioDeviceWindowsCore::AudioDeviceWindowsCore() reinterpret_cast(&_ptrEnumerator)); RTC_DCHECK(_ptrEnumerator); + _ptrEnumerator->RegisterEndpointNotificationCallback(_deviceStateListener); + // DMO initialization for built-in WASAPI AEC. { IMediaObject* ptrDMO = NULL; @@ -499,6 +503,8 @@ AudioDeviceWindowsCore::~AudioDeviceWindowsCore() { Terminate(); + _ptrEnumerator->UnregisterEndpointNotificationCallback(_deviceStateListener); + // The IMMDeviceEnumerator is created during construction. Must release // it here and not in Terminate() since we don't recreate it in Init(). SAFE_RELEASE(_ptrEnumerator); @@ -535,6 +541,11 @@ AudioDeviceWindowsCore::~AudioDeviceWindowsCore() { _hShutdownCaptureEvent = NULL; } + if(NULL != _deviceStateListener) { + delete _deviceStateListener; + _deviceStateListener = NULL; + } + if (_avrtLibrary) { BOOL freeOK = FreeLibrary(_avrtLibrary); if (!freeOK) { @@ -3894,6 +3905,65 @@ int32_t AudioDeviceWindowsCore::_GetDeviceID(IMMDevice* pDevice, return 0; } +int32_t AudioDeviceWindowsCore::SetAudioDeviceSink(AudioDeviceSink* sink) { + _deviceStateListener->SetAudioDeviceSink(sink); + return 0; +} + +void AudioDeviceWindowsCore::DeviceStateListener::SetAudioDeviceSink(AudioDeviceSink *sink) { + callback_ = sink; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::OnDeviceStateChanged(LPCWSTR pwstrDeviceId, DWORD dwNewState) { + RTC_DLOG(LS_INFO) << "AudioDeviceWindowsCore::OnDeviceStateChanged => " << pwstrDeviceId << ", NewState => " << dwNewState; + if(callback_) callback_->OnDevicesUpdated(); + return S_OK; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::OnDeviceAdded(LPCWSTR pwstrDeviceId) { + RTC_DLOG(LS_INFO) << "AudioDeviceWindowsCore::OnDeviceAdded => " << pwstrDeviceId; + return S_OK; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::OnDeviceRemoved(LPCWSTR pwstrDeviceId) { + RTC_DLOG(LS_INFO) << "AudioDeviceWindowsCore::OnDeviceRemoved => " << pwstrDeviceId; + return S_OK; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::OnDefaultDeviceChanged(EDataFlow flow, ERole role, LPCWSTR pwstrDefaultDeviceId) { + RTC_DLOG(LS_INFO) << "AudioDeviceWindowsCore::OnDefaultDeviceChanged => " << pwstrDefaultDeviceId; + return S_OK; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::OnPropertyValueChanged(LPCWSTR pwstrDeviceId, const PROPERTYKEY key) { + //RTC_DLOG(LS_INFO) << "AudioDeviceWindowsCore::OnPropertyValueChanged => " << pwstrDeviceId; + return S_OK; +} + +ULONG AudioDeviceWindowsCore::DeviceStateListener::AddRef() { + ULONG new_ref = InterlockedIncrement(&ref_count_); + // RTC_DLOG(LS_INFO) << "__AddRef => " << new_ref; + return new_ref; +} + +ULONG AudioDeviceWindowsCore::DeviceStateListener::Release() { + ULONG new_ref = InterlockedDecrement(&ref_count_); + // RTC_DLOG(LS_INFO) << "__Release => " << new_ref; + return new_ref; +} + +HRESULT AudioDeviceWindowsCore::DeviceStateListener::QueryInterface(REFIID iid, void** object) { + if (object == nullptr) { + return E_POINTER; + } + if (iid == IID_IUnknown || iid == __uuidof(IMMNotificationClient)) { + *object = static_cast(this); + return S_OK; + } + *object = nullptr; + return E_NOINTERFACE; +} + // ---------------------------------------------------------------------------- // _GetDefaultDevice // ---------------------------------------------------------------------------- diff --git a/modules/audio_device/win/audio_device_core_win.h b/modules/audio_device/win/audio_device_core_win.h index 380effb449..10b6a92b7f 100644 --- a/modules/audio_device/win/audio_device_core_win.h +++ b/modules/audio_device/win/audio_device_core_win.h @@ -22,6 +22,8 @@ #include #include // IMediaObject #include // MMDevice +#include +#include #include "api/scoped_refptr.h" #include "modules/audio_device/audio_device_generic.h" @@ -50,6 +52,34 @@ class AudioDeviceWindowsCore : public AudioDeviceGeneric { AudioDeviceWindowsCore(); ~AudioDeviceWindowsCore(); + class DeviceStateListener : public IMMNotificationClient { + public: + virtual ~DeviceStateListener() = default; + HRESULT __stdcall OnDeviceStateChanged(LPCWSTR pwstrDeviceId, + DWORD dwNewState) override; + HRESULT __stdcall OnDeviceAdded(LPCWSTR pwstrDeviceId) override; + + HRESULT __stdcall OnDeviceRemoved(LPCWSTR pwstrDeviceId) override; + + HRESULT + __stdcall OnDefaultDeviceChanged(EDataFlow flow, + ERole role, + LPCWSTR pwstrDefaultDeviceId) override; + + HRESULT __stdcall OnPropertyValueChanged(LPCWSTR pwstrDeviceId, + const PROPERTYKEY key) override; + // IUnknown (required by IMMNotificationClient). + ULONG __stdcall AddRef() override; + ULONG __stdcall Release() override; + HRESULT __stdcall QueryInterface(REFIID iid, void** object) override; + + void SetAudioDeviceSink(AudioDeviceSink *sink); + + private: + LONG ref_count_ = 1; + AudioDeviceSink *callback_ = nullptr; + }; + static bool CoreAudioIsSupported(); // Retrieve the currently utilized audio layer @@ -150,6 +180,8 @@ class AudioDeviceWindowsCore : public AudioDeviceGeneric { virtual int32_t EnableBuiltInAEC(bool enable); + virtual int32_t SetAudioDeviceSink(AudioDeviceSink* sink); + public: virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); @@ -237,6 +269,7 @@ class AudioDeviceWindowsCore : public AudioDeviceGeneric { IAudioEndpointVolume* _ptrCaptureVolume; ISimpleAudioVolume* _ptrRenderSimpleVolume; + DeviceStateListener *_deviceStateListener = nullptr; // DirectX Media Object (DMO) for the built-in AEC. rtc::scoped_refptr _dmo; rtc::scoped_refptr _mediaBuffer; diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index d31673eb7a..6cbf94f589 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -119,6 +119,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.mm", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", ] @@ -922,6 +923,11 @@ if (is_ios || is_mac) { ] configs += [ "..:no_global_constructors" ] sources = [ + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCAudioDeviceModule+Private.h", + "objc/api/peerconnection/RTCAudioDeviceModule.mm", + "objc/api/peerconnection/RTCIODevice.h", + "objc/api/peerconnection/RTCIODevice.mm", "objc/api/peerconnection/RTCAudioSource+Private.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioSource.mm", @@ -1034,6 +1040,20 @@ if (is_ios || is_mac) { "objc/api/peerconnection/RTCVideoTrack+Private.h", "objc/api/peerconnection/RTCVideoTrack.h", "objc/api/peerconnection/RTCVideoTrack.mm", + "objc/components/audio/RTCAudioBuffer.h", + "objc/components/audio/RTCAudioBuffer.mm", + "objc/components/audio/RTCAudioBuffer+Private.h", + "objc/components/audio/RTCAudioCustomProcessingAdapter.h", + "objc/components/audio/RTCAudioCustomProcessingAdapter.mm", + "objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h", + "objc/components/audio/RTCAudioCustomProcessingDelegate.h", + "objc/components/audio/RTCAudioProcessingConfig.h", + "objc/components/audio/RTCAudioProcessingConfig.mm", + "objc/components/audio/RTCAudioProcessingConfig+Private.h", + "objc/components/audio/RTCAudioProcessingModule.h", + "objc/components/audio/RTCDefaultAudioProcessingModule.h", + "objc/components/audio/RTCDefaultAudioProcessingModule.mm", + "objc/components/audio/RTCDefaultAudioProcessingModule+Private.h", ] configs += [ @@ -1275,6 +1295,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.h", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", "objc/components/audio/RTCAudioDevice.h", "objc/components/audio/RTCAudioSession.h", @@ -1297,6 +1318,8 @@ if (is_ios || is_mac) { "objc/helpers/RTCCameraPreviewView.h", "objc/helpers/RTCDispatcher.h", "objc/helpers/UIDevice+RTCDevice.h", + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCIODevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCConfiguration.h", @@ -1411,6 +1434,8 @@ if (is_ios || is_mac) { output_name = "WebRTC" sources = [ + "objc/api/peerconnection/RTCAudioDeviceModule.h", + "objc/api/peerconnection/RTCIODevice.h", "objc/api/peerconnection/RTCAudioSource.h", "objc/api/peerconnection/RTCAudioTrack.h", "objc/api/peerconnection/RTCCertificate.h", @@ -1477,6 +1502,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrame.h", "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", + "objc/base/RTCAudioRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", "objc/components/capturer/RTCCameraVideoCapturer.h", "objc/components/capturer/RTCFileVideoCapturer.h", diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 8858d21b60..d1276b2ec8 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -261,11 +261,13 @@ if (is_android) { "api/org/webrtc/AudioProcessingFactory.java", "api/org/webrtc/AudioSource.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", "api/org/webrtc/DataChannel.java", "api/org/webrtc/DtmfSender.java", + "api/org/webrtc/ExternalAudioProcessingFactory.java", "api/org/webrtc/FecControllerFactoryFactoryInterface.java", "api/org/webrtc/FrameDecryptor.java", "api/org/webrtc/FrameEncryptor.java", @@ -717,6 +719,8 @@ if (current_os == "linux" || is_android) { "src/jni/pc/add_ice_candidate_observer.cc", "src/jni/pc/add_ice_candidate_observer.h", "src/jni/pc/android_network_monitor.h", + "src/jni/pc/audio_sink.cc", + "src/jni/pc/audio_sink.h", "src/jni/pc/audio_track.cc", "src/jni/pc/call_session_file_rotating_log_sink.cc", "src/jni/pc/crypto_options.cc", @@ -724,6 +728,11 @@ if (current_os == "linux" || is_android) { "src/jni/pc/data_channel.cc", "src/jni/pc/data_channel.h", "src/jni/pc/dtmf_sender.cc", + "src/jni/pc/external_audio_processing_factory.cc", + "src/jni/pc/external_audio_processing_factory.h", + "src/jni/pc/external_audio_processing_interface.h", + "src/jni/pc/external_audio_processor.cc", + "src/jni/pc/external_audio_processor.h", "src/jni/pc/ice_candidate.cc", "src/jni/pc/ice_candidate.h", "src/jni/pc/media_constraints.cc", @@ -1396,11 +1405,13 @@ if (current_os == "linux" || is_android) { sources = [ "api/org/webrtc/AddIceObserver.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", "api/org/webrtc/DataChannel.java", "api/org/webrtc/DtmfSender.java", + "api/org/webrtc/ExternalAudioProcessingFactory.java", "api/org/webrtc/IceCandidate.java", "api/org/webrtc/IceCandidateErrorEvent.java", "api/org/webrtc/MediaConstraints.java", diff --git a/sdk/android/api/org/webrtc/AudioTrack.java b/sdk/android/api/org/webrtc/AudioTrack.java index ca745db634..b30e46cebc 100644 --- a/sdk/android/api/org/webrtc/AudioTrack.java +++ b/sdk/android/api/org/webrtc/AudioTrack.java @@ -10,8 +10,12 @@ package org.webrtc; +import java.util.IdentityHashMap; + /** Java wrapper for a C++ AudioTrackInterface */ public class AudioTrack extends MediaStreamTrack { + private final IdentityHashMap sinks = new IdentityHashMap(); + public AudioTrack(long nativeTrack) { super(nativeTrack); } @@ -23,10 +27,54 @@ public void setVolume(double volume) { nativeSetVolume(getNativeAudioTrack(), volume); } + /** + * Adds an AudioTrackSink to the track. This callback is only + * called for remote audio tracks. + * + * Repeated addSink calls will not add the sink multiple times. + */ + public void addSink(AudioTrackSink sink) { + if (sink == null) { + throw new IllegalArgumentException("The AudioTrackSink is not allowed to be null"); + } + if (!sinks.containsKey(sink)) { + final long nativeSink = nativeWrapSink(sink); + sinks.put(sink, nativeSink); + nativeAddSink(getNativeMediaStreamTrack(), nativeSink); + } + } + + /** + * Removes an AudioTrackSink from the track. + * + * If the AudioTrackSink was not attached to the track, this is a no-op. + */ + public void removeSink(AudioTrackSink sink) { + final Long nativeSink = sinks.remove(sink); + if (nativeSink != null) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + } + + @Override + public void dispose() { + for (long nativeSink : sinks.values()) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + sinks.clear(); + super.dispose(); + } + /** Returns a pointer to webrtc::AudioTrackInterface. */ long getNativeAudioTrack() { return getNativeMediaStreamTrack(); } private static native void nativeSetVolume(long track, double volume); + private static native void nativeAddSink(long track, long nativeSink); + private static native void nativeRemoveSink(long track, long nativeSink); + private static native long nativeWrapSink(AudioTrackSink sink); + private static native void nativeFreeSink(long sink); } diff --git a/sdk/android/api/org/webrtc/AudioTrackSink.java b/sdk/android/api/org/webrtc/AudioTrackSink.java new file mode 100644 index 0000000000..eca390f82c --- /dev/null +++ b/sdk/android/api/org/webrtc/AudioTrackSink.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** + * Java version of rtc::AudioTrackSinkInterface. + */ +public interface AudioTrackSink { + /** + * Implementations should copy the audio data into a local copy if they wish + * to use the data after this function returns. + */ + @CalledByNative + void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate, + int numberOfChannels, int numberOfFrames, + long absoluteCaptureTimestampMs); +} diff --git a/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java b/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java new file mode 100644 index 0000000000..7425d2af57 --- /dev/null +++ b/sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java @@ -0,0 +1,144 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +import androidx.annotation.Nullable; +import org.webrtc.AudioProcessingFactory; + + +public class ExternalAudioProcessingFactory implements AudioProcessingFactory { + + /** + * Interface for external audio processing. + */ + public static interface AudioProcessing { + /** + * Called when the processor should be initialized with a new sample rate and + * number of channels. + */ + @CalledByNative("AudioProcessing") + void initialize(int sampleRateHz, int numChannels); + /** Called when the processor should be reset with a new sample rate. */ + @CalledByNative("AudioProcessing") + void reset(int newRate); + /** + * Processes the given capture or render signal. NOTE: `buffer.data` will be + * freed once this function returns so callers who want to use the data + * asynchronously must make sure to copy it first. + */ + @CalledByNative("AudioProcessing") + void process(int numBands, int numFrames, ByteBuffer buffer); + } + + private long apmPtr; + private long capturePostProcessingPtr; + private long renderPreProcessingPtr; + + public ExternalAudioProcessingFactory() { + apmPtr = nativeGetDefaultApm(); + capturePostProcessingPtr = 0; + renderPreProcessingPtr = 0; + } + + @Override + public long createNative() { + if(apmPtr == 0) { + apmPtr = nativeGetDefaultApm(); + } + return apmPtr; + } + + /** + * Sets the capture post processing module. + * This module is applied to the audio signal after capture and before sending + * to the audio encoder. + */ + public void setCapturePostProcessing(@Nullable AudioProcessing processing) { + checkExternalAudioProcessorExists(); + long newPtr = nativeSetCapturePostProcessing(processing); + if (capturePostProcessingPtr != 0) { + JniCommon.nativeReleaseRef(capturePostProcessingPtr); + capturePostProcessingPtr = 0; + } + capturePostProcessingPtr = newPtr; + } + + /** + * Sets the render pre processing module. + * This module is applied to the audio signal after receiving from the audio + * decoder and before rendering. + */ + public void setRenderPreProcessing(@Nullable AudioProcessing processing) { + checkExternalAudioProcessorExists(); + long newPtr = nativeSetRenderPreProcessing(processing); + if (renderPreProcessingPtr != 0) { + JniCommon.nativeReleaseRef(renderPreProcessingPtr); + renderPreProcessingPtr = 0; + } + renderPreProcessingPtr = newPtr; + } + + /** + * Sets the bypass flag for the capture post processing module. + * If true, the registered audio processing will be bypassed. + */ + public void setBypassFlagForCapturePost( boolean bypass) { + checkExternalAudioProcessorExists(); + nativeSetBypassFlagForCapturePost(bypass); + } + + /** + * Sets the bypass flag for the render pre processing module. + * If true, the registered audio processing will be bypassed. + */ + public void setBypassFlagForRenderPre( boolean bypass) { + checkExternalAudioProcessorExists(); + nativeSetBypassFlagForRenderPre(bypass); + } + + /** + * Destroys the ExternalAudioProcessor. + */ + public void destroy() { + checkExternalAudioProcessorExists(); + if (renderPreProcessingPtr != 0) { + JniCommon.nativeReleaseRef(renderPreProcessingPtr); + renderPreProcessingPtr = 0; + } + if (capturePostProcessingPtr != 0) { + JniCommon.nativeReleaseRef(capturePostProcessingPtr); + capturePostProcessingPtr = 0; + } + nativeDestroy(); + apmPtr = 0; + } + + private void checkExternalAudioProcessorExists() { + if (apmPtr == 0) { + throw new IllegalStateException("ExternalAudioProcessor has been disposed."); + } + } + + private static native long nativeGetDefaultApm(); + private static native long nativeSetCapturePostProcessing(AudioProcessing processing); + private static native long nativeSetRenderPreProcessing(AudioProcessing processing); + private static native void nativeSetBypassFlagForCapturePost(boolean bypass); + private static native void nativeSetBypassFlagForRenderPre(boolean bypass); + private static native void nativeDestroy(); +} diff --git a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java index b118843ea0..8ef45f727b 100644 --- a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -42,6 +42,7 @@ public static class Builder { private AudioTrackErrorCallback audioTrackErrorCallback; private AudioRecordErrorCallback audioRecordErrorCallback; private SamplesReadyCallback samplesReadyCallback; + private PlaybackSamplesReadyCallback playbackSamplesReadyCallback; private AudioTrackStateCallback audioTrackStateCallback; private AudioRecordStateCallback audioRecordStateCallback; private boolean useHardwareAcousticEchoCanceler = isBuiltInAcousticEchoCancelerSupported(); @@ -140,6 +141,14 @@ public Builder setSamplesReadyCallback(SamplesReadyCallback samplesReadyCallback return this; } + /** + * Set a callback to listen to the audio output passed to the AudioTrack. + */ + public Builder setPlaybackSamplesReadyCallback(PlaybackSamplesReadyCallback playbackSamplesReadyCallback) { + this.playbackSamplesReadyCallback = playbackSamplesReadyCallback; + return this; + } + /** * Set a callback to retrieve information from the AudioTrack on when audio starts and stop. */ @@ -258,7 +267,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() { samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); final WebRtcAudioTrack audioOutput = new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, - audioTrackStateCallback, useLowLatency, enableVolumeLogger); + audioTrackStateCallback, playbackSamplesReadyCallback, useLowLatency, enableVolumeLogger); return new JavaAudioDeviceModule(context, audioManager, audioInput, audioOutput, inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput); } @@ -325,6 +334,11 @@ public static interface SamplesReadyCallback { void onWebRtcAudioRecordSamplesReady(AudioSamples samples); } + /** Called when new audio samples are ready. This should only be set for debug purposes */ + public static interface PlaybackSamplesReadyCallback { + void onWebRtcAudioTrackSamplesReady(AudioSamples samples); + } + /* AudioTrack */ // Audio playout/track error handler functions. public enum AudioTrackStartErrorCode { @@ -362,8 +376,8 @@ public static boolean isBuiltInNoiseSuppressorSupported() { private final Context context; private final AudioManager audioManager; - private final WebRtcAudioRecord audioInput; - private final WebRtcAudioTrack audioOutput; + public final WebRtcAudioRecord audioInput; + public final WebRtcAudioTrack audioOutput; private final int inputSampleRate; private final int outputSampleRate; private final boolean useStereoInput; diff --git a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java index 2b34e34013..25d10e4f61 100644 --- a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java +++ b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -20,12 +20,14 @@ import android.os.Process; import androidx.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Arrays; import org.webrtc.CalledByNative; import org.webrtc.Logging; import org.webrtc.ThreadUtils; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback; +import org.webrtc.audio.JavaAudioDeviceModule.PlaybackSamplesReadyCallback; import org.webrtc.audio.LowLatencyAudioBufferManager; class WebRtcAudioTrack { @@ -62,7 +64,7 @@ class WebRtcAudioTrack { private ByteBuffer byteBuffer; - private @Nullable final AudioAttributes audioAttributes; + public @Nullable AudioAttributes audioAttributes; private @Nullable AudioTrack audioTrack; private @Nullable AudioTrackThread audioThread; private final VolumeLogger volumeLogger; @@ -76,6 +78,7 @@ class WebRtcAudioTrack { private final @Nullable AudioTrackErrorCallback errorCallback; private final @Nullable AudioTrackStateCallback stateCallback; + private final @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback; /** * Audio thread which keeps calling AudioTrack.write() to stream audio. @@ -129,6 +132,17 @@ public void run() { reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten); } } + + if (audioSamplesReadyCallback != null && keepAlive) { + // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily + // at index 0. + byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), + sizeInBytes + byteBuffer.arrayOffset()); + audioSamplesReadyCallback.onWebRtcAudioTrackSamplesReady( + new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), + audioTrack.getChannelCount(), audioTrack.getSampleRate(), data)); + } + if (useLowLatency) { bufferManager.maybeAdjustBufferSize(audioTrack); } @@ -154,13 +168,13 @@ public void stopThread() { @CalledByNative WebRtcAudioTrack(Context context, AudioManager audioManager) { this(context, audioManager, null /* audioAttributes */, null /* errorCallback */, - null /* stateCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); + null /* stateCallback */, null /* audioSamplesReadyCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); } WebRtcAudioTrack(Context context, AudioManager audioManager, @Nullable AudioAttributes audioAttributes, @Nullable AudioTrackErrorCallback errorCallback, - @Nullable AudioTrackStateCallback stateCallback, boolean useLowLatency, - boolean enableVolumeLogger) { + @Nullable AudioTrackStateCallback stateCallback, @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback, + boolean useLowLatency, boolean enableVolumeLogger) { threadChecker.detachThread(); this.context = context; this.audioManager = audioManager; @@ -168,6 +182,7 @@ public void stopThread() { this.errorCallback = errorCallback; this.stateCallback = stateCallback; this.volumeLogger = enableVolumeLogger ? new VolumeLogger(audioManager) : null; + this.audioSamplesReadyCallback = audioSamplesReadyCallback; this.useLowLatency = useLowLatency; Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); } diff --git a/sdk/android/src/jni/pc/audio_sink.cc b/sdk/android/src/jni/pc/audio_sink.cc new file mode 100644 index 0000000000..5bd88c75f6 --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/android/src/jni/pc/audio_sink.h" + +#include "sdk/android/generated_peerconnection_jni/AudioTrackSink_jni.h" + +namespace webrtc { +namespace jni { + +AudioTrackSinkWrapper::AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink) + : j_sink_(jni, j_sink) {} + +AudioTrackSinkWrapper::~AudioTrackSinkWrapper() {} + +void AudioTrackSinkWrapper::OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + int length = (bits_per_sample / 8) * number_of_channels * number_of_frames; + ScopedJavaLocalRef audio_buffer = + NewDirectByteBuffer(jni, (void *) audio_data, length); + Java_AudioTrackSink_onData(jni, j_sink_, + audio_buffer, bits_per_sample, sample_rate, (int) number_of_channels, (int) number_of_frames, (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0)); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/audio_sink.h b/sdk/android/src/jni/pc/audio_sink.h new file mode 100644 index 0000000000..809f460e0c --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.h @@ -0,0 +1,41 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ +#define SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ + +#include + +#include "api/media_stream_interface.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +class AudioTrackSinkWrapper : public webrtc::AudioTrackSinkInterface { + public: + AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink); + ~AudioTrackSinkWrapper() override; + + private: + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override; + + const ScopedJavaGlobalRef j_sink_; +}; + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ diff --git a/sdk/android/src/jni/pc/audio_track.cc b/sdk/android/src/jni/pc/audio_track.cc index 36ed43f1d4..df2d605893 100644 --- a/sdk/android/src/jni/pc/audio_track.cc +++ b/sdk/android/src/jni/pc/audio_track.cc @@ -9,6 +9,8 @@ */ #include "api/media_stream_interface.h" +#include "sdk/android/src/jni/pc/audio_sink.h" + #include "sdk/android/generated_peerconnection_jni/AudioTrack_jni.h" namespace webrtc { @@ -20,5 +22,29 @@ static void JNI_AudioTrack_SetVolume(JNIEnv*, jlong j_p, jdouble volume) { source->SetVolume(volume); } +static void JNI_AudioTrack_AddSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->AddSink(reinterpret_cast(j_native_sink)); +} + +static void JNI_AudioTrack_RemoveSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->RemoveSink(reinterpret_cast(j_native_sink)); +} + +static jlong JNI_AudioTrack_WrapSink(JNIEnv* jni, + const JavaParamRef& sink) { + return jlongFromPointer(new AudioTrackSinkWrapper(jni, sink)); +} + +static void JNI_AudioTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { + delete reinterpret_cast(j_native_sink); +} + + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_factory.cc b/sdk/android/src/jni/pc/external_audio_processing_factory.cc new file mode 100644 index 0000000000..3d7ee7a4d9 --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_factory.cc @@ -0,0 +1,143 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/android/src/jni/pc/external_audio_processing_factory.h" + +#include +#include + +#include "api/make_ref_counted.h" +#include "rtc_base/ref_counted_object.h" +#include "sdk/android/generated_peerconnection_jni/ExternalAudioProcessingFactory_jni.h" +#include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/pc/external_audio_processor.h" + +namespace webrtc { +namespace jni { + +ExternalAudioProcessingJni::ExternalAudioProcessingJni( + JNIEnv* jni, + const JavaRef& j_processing) + : j_processing_global_(jni, j_processing) {} +ExternalAudioProcessingJni::~ExternalAudioProcessingJni() {} +void ExternalAudioProcessingJni::Initialize(int sample_rate_hz, + int num_channels) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + Java_AudioProcessing_initialize(env, j_processing_global_, sample_rate_hz, + num_channels); +} + +void ExternalAudioProcessingJni::Reset(int new_rate) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + Java_AudioProcessing_reset(env, j_processing_global_, new_rate); +} + +void ExternalAudioProcessingJni::Process(int num_bands, int num_frames, int buffer_size, float* buffer) { + JNIEnv* env = AttachCurrentThreadIfNeeded(); + ScopedJavaLocalRef audio_buffer = + NewDirectByteBuffer(env, (void*)buffer, buffer_size * sizeof(float)); + Java_AudioProcessing_process(env, j_processing_global_, num_bands, num_frames, audio_buffer); +} + +ExternalAudioProcessingFactory::ExternalAudioProcessingFactory() { + capture_post_processor_ = new ExternalAudioProcessor(); + std::unique_ptr capture_post_processor( + capture_post_processor_); + + render_pre_processor_ = new ExternalAudioProcessor(); + std::unique_ptr render_pre_processor( + render_pre_processor_); + + apm_ = webrtc::AudioProcessingBuilder() + .SetCapturePostProcessing(std::move(capture_post_processor)) + .SetRenderPreProcessing(std::move(render_pre_processor)) + .Create(); + + webrtc::AudioProcessing::Config config; + apm_->ApplyConfig(config); +} + +static ExternalAudioProcessingFactory* default_processor_ptr; + +static jlong JNI_ExternalAudioProcessingFactory_GetDefaultApm(JNIEnv* env) { + if (!default_processor_ptr) { + auto default_processor = rtc::make_ref_counted(); + default_processor_ptr = default_processor.release(); + } + return webrtc::jni::jlongFromPointer(default_processor_ptr->apm().get()); +} + +static jlong JNI_ExternalAudioProcessingFactory_SetCapturePostProcessing( + JNIEnv* env, + const JavaParamRef& j_processing) { + if (!default_processor_ptr) { + return 0; + } + auto processing = + rtc::make_ref_counted(env, j_processing); + processing->AddRef(); + default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing( + processing.get()); + return jlongFromPointer(processing.get()); +} + +static jlong JNI_ExternalAudioProcessingFactory_SetRenderPreProcessing( + JNIEnv* env, + const JavaParamRef& j_processing) { + if (!default_processor_ptr) { + return 0; + } + auto processing = + rtc::make_ref_counted(env, j_processing); + processing->AddRef(); + default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing( + processing.get()); + return jlongFromPointer(processing.get()); +} + +static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForCapturePost( + JNIEnv* env, + jboolean bypass) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->capture_post_processor()->SetBypassFlag(bypass); +} + +static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForRenderPre( + JNIEnv* env, + jboolean bypass) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->render_pre_processor()->SetBypassFlag(bypass); +} + +static void JNI_ExternalAudioProcessingFactory_Destroy(JNIEnv* env) { + if (!default_processor_ptr) { + return; + } + default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing( + nullptr); + default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing( + nullptr); + delete default_processor_ptr; +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_factory.h b/sdk/android/src/jni/pc/external_audio_processing_factory.h new file mode 100644 index 0000000000..5dfebe81fc --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_factory.h @@ -0,0 +1,68 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define WEBRTC_APM_DEBUG_DUMP 0 + +#include "rtc_base/ref_counted_object.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" +#include "sdk/android/src/jni/pc/external_audio_processor.h" +#include "sdk/android/src/jni/pc/external_audio_processing_interface.h" + +namespace webrtc { +namespace jni { + +class ExternalAudioProcessingJni + : public webrtc::ExternalAudioProcessingInterface, + public rtc::RefCountInterface { + public: + ExternalAudioProcessingJni(JNIEnv* jni, const JavaRef& j_processing); + ~ExternalAudioProcessingJni(); + + protected: + virtual void Initialize(int sample_rate_hz, int num_channels) override; + virtual void Reset(int new_rate) override; + virtual void Process(int num_bans, int num_frames, int buffer_size, float* buffer) override; + + private: + const ScopedJavaGlobalRef j_processing_global_; + const ScopedJavaGlobalRef j_processing_; +}; + +class ExternalAudioProcessingFactory : public rtc::RefCountInterface { + public: + ExternalAudioProcessingFactory(); + virtual ~ExternalAudioProcessingFactory() = default; + + ExternalAudioProcessor* capture_post_processor() { + return capture_post_processor_; + } + + ExternalAudioProcessor* render_pre_processor() { + return render_pre_processor_; + } + + rtc::scoped_refptr apm() { return apm_; } + + private: + rtc::scoped_refptr apm_; + ExternalAudioProcessor* capture_post_processor_; + ExternalAudioProcessor* render_pre_processor_; +}; + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processing_interface.h b/sdk/android/src/jni/pc/external_audio_processing_interface.h new file mode 100644 index 0000000000..1202be106b --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processing_interface.h @@ -0,0 +1,33 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ +#define API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ + +namespace webrtc { + +class ExternalAudioProcessingInterface { + public: + virtual void Initialize(int sample_rate_hz, int num_channels) = 0; + virtual void Reset(int new_rate) = 0; + virtual void Process(int num_bands, int num_frames, int buffer_size, float* buffer) = 0; + + protected: + virtual ~ExternalAudioProcessingInterface() = default; +}; + +} // namespace webrtc + +#endif // API_ANDROID_JNI_EXTERNALAUDIOPROCESSORINTERFACE_H_ diff --git a/sdk/android/src/jni/pc/external_audio_processor.cc b/sdk/android/src/jni/pc/external_audio_processor.cc new file mode 100644 index 0000000000..274982d6d4 --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processor.cc @@ -0,0 +1,72 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/android/src/jni/pc/external_audio_processor.h" + +namespace webrtc { + +void ExternalAudioProcessor::SetExternalAudioProcessing( + ExternalAudioProcessingInterface* processor) { + webrtc::MutexLock lock(&mutex_); + external_processor_ = processor; + if (initialized_) { + external_processor_->Initialize(sample_rate_hz_, num_channels_); + } +} + +void ExternalAudioProcessor::SetBypassFlag(bool bypass) { + webrtc::MutexLock lock(&mutex_); + bypass_flag_ = bypass; +} + +void ExternalAudioProcessor::Initialize(int sample_rate_hz, int num_channels) { + webrtc::MutexLock lock(&mutex_); + sample_rate_hz_ = sample_rate_hz; + num_channels_ = num_channels; + if (external_processor_) { + external_processor_->Initialize(sample_rate_hz, num_channels); + } + initialized_ = true; +} + +void ExternalAudioProcessor::Process(webrtc::AudioBuffer* audio) { + webrtc::MutexLock lock(&mutex_); + if (!external_processor_ || bypass_flag_ || !initialized_) { + return; + } + + size_t num_frames = audio->num_frames(); + size_t num_bands =audio->num_bands(); + + // 1 buffer = 10ms of frames + int rate = num_frames * 100; + + if (rate != sample_rate_hz_) { + external_processor_->Reset(rate); + sample_rate_hz_ = rate; + } + + external_processor_->Process(num_bands, num_frames, kNsFrameSize * num_bands, audio->channels()[0]); +} + +std::string ExternalAudioProcessor::ToString() const { + return "ExternalAudioProcessor"; +} + +void ExternalAudioProcessor::SetRuntimeSetting( + webrtc::AudioProcessing::RuntimeSetting setting) {} + +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/external_audio_processor.h b/sdk/android/src/jni/pc/external_audio_processor.h new file mode 100644 index 0000000000..1dc31809fc --- /dev/null +++ b/sdk/android/src/jni/pc/external_audio_processor.h @@ -0,0 +1,57 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ +#define SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ + +#define WEBRTC_APM_DEBUG_DUMP 0 + +#include "modules/audio_processing/audio_buffer.h" +#include "modules/audio_processing/audio_processing_impl.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "sdk/android/src/jni/pc/external_audio_processing_interface.h" + +namespace webrtc { + +class ExternalAudioProcessor : public webrtc::CustomProcessing { + public: + ExternalAudioProcessor() = default; + ~ExternalAudioProcessor() override = default; + + void SetExternalAudioProcessing( + ExternalAudioProcessingInterface* processor); + + void SetBypassFlag(bool bypass); + + private: + void Initialize(int sample_rate_hz, int num_channels) override; + void Process(webrtc::AudioBuffer* audio) override; + std::string ToString() const override; + void SetRuntimeSetting( + webrtc::AudioProcessing::RuntimeSetting setting) override; + + private: + mutable webrtc::Mutex mutex_; + ExternalAudioProcessingInterface* external_processor_; + bool bypass_flag_ = false; + bool initialized_ = false; + int sample_rate_hz_ = 0; + int num_channels_ = 0; +}; + +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_PC_EXTERNAL_AUDIO_PROCESSOR_H_ diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h new file mode 100644 index 0000000000..4eb91b93c7 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule+Private.h @@ -0,0 +1,31 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioDeviceModule.h" +#import "sdk/objc/native/api/audio_device_module.h" + +#include "rtc_base/thread.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCAudioDeviceModule () + +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module + workerThread:(rtc::Thread *)workerThread; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h new file mode 100644 index 0000000000..1a9e339bd6 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.h @@ -0,0 +1,56 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCMacros.h" +#import "RTCIODevice.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef void (^RTCOnAudioDevicesDidUpdate)(); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioDeviceModule) : NSObject + +@property(nonatomic, readonly) NSArray *outputDevices; +@property(nonatomic, readonly) NSArray *inputDevices; + +@property(nonatomic, readonly) BOOL playing; +@property(nonatomic, readonly) BOOL recording; + +@property(nonatomic, assign) RTC_OBJC_TYPE(RTCIODevice) *outputDevice; +@property(nonatomic, assign) RTC_OBJC_TYPE(RTCIODevice) *inputDevice; + +// Executes low-level API's in sequence to switch the device +// Use outputDevice / inputDevice property unless you need to know if setting the device is +// successful. +- (BOOL)trySetOutputDevice:(nullable RTCIODevice *)device; +- (BOOL)trySetInputDevice:(nullable RTCIODevice *)device; + +- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler; + +- (BOOL)startPlayout; +- (BOOL)stopPlayout; +- (BOOL)initPlayout; +- (BOOL)startRecording; +- (BOOL)stopRecording; +- (BOOL)initRecording; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm new file mode 100644 index 0000000000..5c116fae53 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCAudioDeviceModule.mm @@ -0,0 +1,294 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#import "RTCAudioDeviceModule.h" +#import "RTCAudioDeviceModule+Private.h" +#import "RTCIODevice+Private.h" +#import "base/RTCLogging.h" + +#import "sdk/objc/native/api/audio_device_module.h" + +class AudioDeviceSink : public webrtc::AudioDeviceSink { + public: + AudioDeviceSink() {} + + void OnDevicesUpdated() override { + + RTCLogInfo(@"AudioDeviceSink OnDevicesUpdated"); + + if (callback_handler_) { + callback_handler_(); + } + } + + // private: + RTCOnAudioDevicesDidUpdate callback_handler_; +}; + +@implementation RTC_OBJC_TYPE (RTCAudioDeviceModule) { + rtc::Thread *_workerThread; + rtc::scoped_refptr _native; + AudioDeviceSink *_sink; +} + +- (instancetype)initWithNativeModule:(rtc::scoped_refptr )module + workerThread:(rtc::Thread * )workerThread { + + RTCLogInfo(@"RTCAudioDeviceModule initWithNativeModule:workerThread:"); + + self = [super init]; + _native = module; + _workerThread = workerThread; + + _sink = new AudioDeviceSink(); + + _workerThread->BlockingCall([self] { + _native->SetAudioDeviceSink(_sink); + }); + + return self; +} + +- (NSArray *)outputDevices { + + return _workerThread->BlockingCall([self] { + return [self _outputDevices]; + }); +} + +- (NSArray *)inputDevices { + return _workerThread->BlockingCall([self] { + return [self _inputDevices]; + }); +} + +- (RTCIODevice *)outputDevice { + return _workerThread->BlockingCall([self] { + + NSArray *devices = [self _outputDevices]; + int16_t devicesCount = (int16_t)([devices count]); + int16_t index = _native->GetPlayoutDevice(); + + if (devicesCount == 0 || index <= -1 || index > (devicesCount - 1)) { + return (RTC_OBJC_TYPE(RTCIODevice) *)nil; + } + + return (RTC_OBJC_TYPE(RTCIODevice) *)[devices objectAtIndex:index]; + }); +} + +- (void)setOutputDevice: (RTCIODevice *)device { + [self trySetOutputDevice: device]; +} + +- (BOOL)trySetOutputDevice: (RTCIODevice *)device { + + return _workerThread->BlockingCall([self, device] { + + NSUInteger index = 0; + NSArray *devices = [self _outputDevices]; + + if ([devices count] == 0) { + return NO; + } + + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCIODevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); + }]; + if (index == NSNotFound) { + return NO; + } + } + + _native->StopPlayout(); + + if (_native->SetPlayoutDevice(index) == 0 + && _native->InitPlayout() == 0 + && _native->StartPlayout() == 0) { + + return YES; + } + + return NO; + }); +} + +- (RTCIODevice *)inputDevice { + + return _workerThread->BlockingCall([self] { + + NSArray *devices = [self _inputDevices]; + int16_t devicesCount = (int16_t)([devices count]); + int16_t index = _native->GetRecordingDevice(); + + if (devicesCount == 0 || index <= -1 || index > (devicesCount - 1)) { + return (RTC_OBJC_TYPE(RTCIODevice) *)nil; + } + + return (RTC_OBJC_TYPE(RTCIODevice) *)[devices objectAtIndex:index]; + }); +} + +- (void)setInputDevice: (RTCIODevice *)device { + [self trySetInputDevice: device]; +} + +- (BOOL)trySetInputDevice: (RTCIODevice *)device { + + return _workerThread->BlockingCall([self, device] { + + NSUInteger index = 0; + NSArray *devices = [self _inputDevices]; + + if ([devices count] == 0) { + return NO; + } + + if (device != nil) { + index = [devices indexOfObjectPassingTest:^BOOL(RTCIODevice *e, NSUInteger i, BOOL *stop) { + return (*stop = [e.deviceId isEqualToString:device.deviceId]); + }]; + if (index == NSNotFound) { + return NO; + } + } + + _native->StopRecording(); + + if (_native->SetRecordingDevice(index) == 0 + && _native->InitRecording() == 0 + && _native->StartRecording() == 0) { + + return YES; + } + + return NO; + }); +} + +- (BOOL)playing { + + return _workerThread->BlockingCall([self] { + return _native->Playing(); + }); +} + +- (BOOL)recording { + + return _workerThread->BlockingCall([self] { + return _native->Recording(); + }); +} + +#pragma mark - Low-level access + +- (BOOL)startPlayout { + + return _workerThread->BlockingCall([self] { + return _native->StartPlayout() == 0; + }); +} + +- (BOOL)stopPlayout { + + return _workerThread->BlockingCall([self] { + return _native->StopPlayout() == 0; + }); +} + +- (BOOL)initPlayout { + + return _workerThread->BlockingCall([self] { + return _native->InitPlayout() == 0; + }); +} + +- (BOOL)startRecording { + + return _workerThread->BlockingCall([self] { + return _native->StartRecording() == 0; + }); +} + +- (BOOL)stopRecording { + + return _workerThread->BlockingCall([self] { + return _native->StopRecording() == 0; + }); +} + +- (BOOL)initRecording { + + return _workerThread->BlockingCall([self] { + return _native->InitRecording() == 0; + }); +} + +- (BOOL)setDevicesUpdatedHandler: (nullable RTCOnAudioDevicesDidUpdate) handler { + _sink->callback_handler_ = handler; + return YES; +} + +#pragma mark - Private + +- (NSArray *)_outputDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _native->PlayoutDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _native->PlayoutDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCIODevice *device = [[RTCIODevice alloc] initWithType:RTCIODeviceTypeOutput deviceId:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; +} + +- (NSArray *)_inputDevices { + + char guid[webrtc::kAdmMaxGuidSize + 1] = {0}; + char name[webrtc::kAdmMaxDeviceNameSize + 1] = {0}; + + NSMutableArray *result = [NSMutableArray array]; + + int16_t count = _native->RecordingDevices(); + + if (count > 0) { + for (int i = 0; i < count; i++) { + _native->RecordingDeviceName(i, name, guid); + NSString *strGUID = [[NSString alloc] initWithCString:guid encoding:NSUTF8StringEncoding]; + NSString *strName = [[NSString alloc] initWithCString:name encoding:NSUTF8StringEncoding]; + RTCIODevice *device = [[RTCIODevice alloc] initWithType:RTCIODeviceTypeInput deviceId:strGUID name:strName]; + [result addObject: device]; + } + } + + return result; +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h index 6495500484..38c0bd3b1b 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack+Private.h @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import #import "RTCAudioTrack.h" #include "api/media_stream_interface.h" @@ -15,17 +16,18 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCPeerConnectionFactory); -@interface RTC_OBJC_TYPE (RTCAudioTrack) -() +@interface RTC_OBJC_TYPE (RTCAudioTrack) () - /** AudioTrackInterface created or passed in at construction. */ - @property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; +/** AudioTrackInterface created or passed in at construction. */ +@property(nonatomic, readonly) rtc::scoped_refptr nativeAudioTrack; /** Initialize an RTCAudioTrack with an id. */ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)factory source:(RTC_OBJC_TYPE(RTCAudioSource) *)source trackId:(NSString *)trackId; +- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.h b/sdk/objc/api/peerconnection/RTCAudioTrack.h index 95eb5d3d48..c8218ad926 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.h +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.h @@ -13,6 +13,7 @@ NS_ASSUME_NONNULL_BEGIN +@protocol RTC_OBJC_TYPE (RTCAudioRenderer); @class RTC_OBJC_TYPE(RTCAudioSource); RTC_OBJC_EXPORT @@ -23,6 +24,13 @@ RTC_OBJC_EXPORT /** The audio source for this audio track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCAudioSource) * source; +/** Register a renderer that will receive all audio CMSampleBuffers on this track. + * Does not retain. */ +- (void)addRenderer:(id)renderer; + +/** Deregister a renderer */ +- (void)removeRenderer:(id)renderer; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 5c1736f436..62db8b46c5 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -8,8 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import +#import + #import "RTCAudioTrack+Private.h" +#import "RTCAudioRenderer.h" #import "RTCAudioSource+Private.h" #import "RTCMediaStreamTrack+Private.h" #import "RTCPeerConnectionFactory+Private.h" @@ -17,7 +21,170 @@ #include "rtc_base/checks.h" -@implementation RTC_OBJC_TYPE (RTCAudioTrack) +namespace webrtc { +/** + * Captures audio data and converts to CMSampleBuffers + */ +class AudioSinkConverter : public rtc::RefCountInterface, public webrtc::AudioTrackSinkInterface { + private: + os_unfair_lock *lock_; + __weak RTCAudioTrack *audio_track_; + int64_t total_frames_ = 0; + bool attached_ = false; + + public: + AudioSinkConverter(RTCAudioTrack *audioTrack, os_unfair_lock *lock) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter init"; + audio_track_ = audioTrack; + lock_ = lock; + } + + ~AudioSinkConverter() { + // + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter dealloc"; + } + + // Must be called while locked + void TryAttach() { + if (attached_) { + // Already attached + return; + } + RTC_LOG(LS_INFO) << "RTCAudioTrack attaching sink..."; + // Reset for creating CMSampleTimingInfo correctly + audio_track_.nativeAudioTrack->AddSink(this); + total_frames_ = 0; + attached_ = true; + } + + // Must be called while locked + void TryDetach() { + if (!attached_) { + // Already detached + return; + } + RTC_LOG(LS_INFO) << "RTCAudioTrack detaching sink..."; + audio_track_.nativeAudioTrack->RemoveSink(this); + attached_ = false; + } + + void OnData(const void *audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData bits_per_sample: " + << bits_per_sample << " sample_rate: " << sample_rate + << " number_of_channels: " << number_of_channels + << " number_of_frames: " << number_of_frames + << " absolute_capture_timestamp_ms: " + << (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0); + + bool is_locked = os_unfair_lock_trylock(lock_); + if (!is_locked) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already locked, skipping..."; + return; + } + bool is_attached = attached_; + os_unfair_lock_unlock(lock_); + + if (!is_attached) { + RTC_LOG(LS_INFO) << "RTCAudioTrack.AudioSinkConverter OnData already detached, skipping..."; + return; + } + + /* + * Convert to CMSampleBuffer + */ + + if (!(number_of_channels == 1 || number_of_channels == 2)) { + NSLog(@"RTCAudioTrack: Only mono or stereo is supported currently. numberOfChannels: %zu", + number_of_channels); + return; + } + + OSStatus status; + + AudioChannelLayout acl; + bzero(&acl, sizeof(acl)); + acl.mChannelLayoutTag = + number_of_channels == 2 ? kAudioChannelLayoutTag_Stereo : kAudioChannelLayoutTag_Mono; + + AudioStreamBasicDescription sd; + sd.mSampleRate = sample_rate; + sd.mFormatID = kAudioFormatLinearPCM; + sd.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; + sd.mFramesPerPacket = 1; + sd.mChannelsPerFrame = number_of_channels; + sd.mBitsPerChannel = bits_per_sample; /* 16 */ + sd.mBytesPerFrame = sd.mChannelsPerFrame * (sd.mBitsPerChannel / 8); + sd.mBytesPerPacket = sd.mBytesPerFrame; + + CMSampleTimingInfo timing = { + CMTimeMake(1, sample_rate), + CMTimeMake(total_frames_, sample_rate), + kCMTimeInvalid, + }; + + total_frames_ += number_of_frames; // update the total + + CMFormatDescriptionRef format = NULL; + status = CMAudioFormatDescriptionCreate( + kCFAllocatorDefault, &sd, sizeof(acl), &acl, 0, NULL, NULL, &format); + + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to create audio format description"); + return; + } + + CMSampleBufferRef buffer; + status = CMSampleBufferCreate(kCFAllocatorDefault, + NULL, + false, + NULL, + NULL, + format, + (CMItemCount)number_of_frames, + 1, + &timing, + 0, + NULL, + &buffer); + // format is no longer required + CFRelease(format); + + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); + return; + } + + AudioBufferList bufferList; + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0].mNumberChannels = sd.mChannelsPerFrame; + bufferList.mBuffers[0].mDataByteSize = (UInt32)(number_of_frames * sd.mBytesPerFrame); + bufferList.mBuffers[0].mData = (void *)audio_data; + status = CMSampleBufferSetDataBufferFromAudioBufferList( + buffer, kCFAllocatorDefault, kCFAllocatorDefault, 0, &bufferList); + if (status != 0) { + NSLog(@"RTCAudioTrack: Failed to convert audio buffer list into sample buffer"); + return; + } + + // Report back to RTCAudioTrack + [audio_track_ didCaptureSampleBuffer:buffer]; + + CFRelease(buffer); + } +}; +} // namespace webrtc + +@implementation RTC_OBJC_TYPE (RTCAudioTrack) { + rtc::scoped_refptr _audioConverter; + // Stores weak references to renderers + NSHashTable *_renderers; + os_unfair_lock _lock; +} @synthesize source = _source; @@ -43,7 +210,21 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto NSParameterAssert(factory); NSParameterAssert(nativeTrack); NSParameterAssert(type == RTCMediaStreamTrackTypeAudio); - return [super initWithFactory:factory nativeTrack:nativeTrack type:type]; + if (self = [super initWithFactory:factory nativeTrack:nativeTrack type:type]) { + RTC_LOG(LS_INFO) << "RTCAudioTrack init"; + _renderers = [NSHashTable weakObjectsHashTable]; + _audioConverter = new rtc::RefCountedObject(self, &_lock); + } + + return self; +} + +- (void)dealloc { + os_unfair_lock_lock(&_lock); + _audioConverter->TryDetach(); + os_unfair_lock_unlock(&_lock); + + RTC_LOG(LS_INFO) << "RTCAudioTrack dealloc"; } - (RTC_OBJC_TYPE(RTCAudioSource) *)source { @@ -57,6 +238,25 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto return _source; } +- (void)addRenderer:(id)renderer { + os_unfair_lock_lock(&_lock); + [_renderers addObject:renderer]; + _audioConverter->TryAttach(); + os_unfair_lock_unlock(&_lock); +} + +- (void)removeRenderer:(id)renderer { + os_unfair_lock_lock(&_lock); + [_renderers removeObject:renderer]; + NSUInteger renderersCount = _renderers.allObjects.count; + + if (renderersCount == 0) { + // Detach if no more renderers... + _audioConverter->TryDetach(); + } + os_unfair_lock_unlock(&_lock); +} + #pragma mark - Private - (rtc::scoped_refptr)nativeAudioTrack { @@ -64,4 +264,18 @@ - (instancetype)initWithFactory:(RTC_OBJC_TYPE(RTCPeerConnectionFactory) *)facto static_cast(self.nativeTrack.get())); } +- (void)didCaptureSampleBuffer:(CMSampleBufferRef)sampleBuffer { + bool is_locked = os_unfair_lock_trylock(&_lock); + if (!is_locked) { + RTC_LOG(LS_INFO) << "RTCAudioTrack didCaptureSampleBuffer already locked, skipping..."; + return; + } + NSArray *renderers = [_renderers allObjects]; + os_unfair_lock_unlock(&_lock); + + for (id renderer in renderers) { + [renderer renderSampleBuffer:sampleBuffer]; + } +} + @end diff --git a/sdk/objc/api/peerconnection/RTCIODevice+Private.h b/sdk/objc/api/peerconnection/RTCIODevice+Private.h new file mode 100644 index 0000000000..0eb09b83a4 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCIODevice+Private.h @@ -0,0 +1,28 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#import "RTCIODevice.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCIODevice () + +- (instancetype)initWithType:(RTCIODeviceType)type + deviceId:(NSString *)deviceId + name:(NSString* )name; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCIODevice.h b/sdk/objc/api/peerconnection/RTCIODevice.h new file mode 100644 index 0000000000..f44d532081 --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCIODevice.h @@ -0,0 +1,41 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef NS_ENUM(NSInteger, RTCIODeviceType) { + RTCIODeviceTypeOutput, + RTCIODeviceTypeInput, +}; + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE(RTCIODevice) : NSObject + ++ (instancetype)defaultDeviceWithType: (RTCIODeviceType)type; +- (instancetype)init NS_UNAVAILABLE; + +@property(nonatomic, readonly) BOOL isDefault; +@property(nonatomic, readonly) RTCIODeviceType type; +@property(nonatomic, copy, readonly) NSString *deviceId; +@property(nonatomic, copy, readonly) NSString *name; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCIODevice.mm b/sdk/objc/api/peerconnection/RTCIODevice.mm new file mode 100644 index 0000000000..27e1255e8e --- /dev/null +++ b/sdk/objc/api/peerconnection/RTCIODevice.mm @@ -0,0 +1,71 @@ +/* + * Copyright 2022 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCIODevice.h" +#import "RTCIODevice+Private.h" + +NSString *const kDefaultDeviceId = @"default"; + +@implementation RTCIODevice + +@synthesize type = _type; +@synthesize deviceId = _deviceId; +@synthesize name = _name; + ++ (instancetype)defaultDeviceWithType: (RTCIODeviceType)type { + return [[self alloc] initWithType: type + deviceId: kDefaultDeviceId + name: @""]; +} + +- (instancetype)initWithType: (RTCIODeviceType)type + deviceId: (NSString *)deviceId + name: (NSString* )name { + if (self = [super init]) { + _type = type; + _deviceId = deviceId; + _name = name; + } + return self; +} + +#pragma mark - IODevice + +- (BOOL)isDefault { + return [_deviceId isEqualToString: kDefaultDeviceId]; +} + +#pragma mark - Equatable + +- (BOOL)isEqual: (id)object { + if (self == object) { + return YES; + } + if (object == nil) { + return NO; + } + if (![object isMemberOfClass:[self class]]) { + return NO; + } + + return [_deviceId isEqualToString:((RTC_OBJC_TYPE(RTCIODevice) *)object).deviceId]; +} + +- (NSUInteger)hash { + return [_deviceId hash]; +} + +@end diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h index f361b9f0ea..cc45aba1ec 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory+Native.h @@ -50,7 +50,8 @@ NS_ASSUME_NONNULL_BEGIN audioDeviceModule: (nullable webrtc::AudioDeviceModule *)audioDeviceModule audioProcessingModule: - (rtc::scoped_refptr)audioProcessingModule; + (rtc::scoped_refptr)audioProcessingModule + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing; - (instancetype) initWithNativeAudioEncoderFactory: @@ -65,7 +66,8 @@ NS_ASSUME_NONNULL_BEGIN audioProcessingModule: (rtc::scoped_refptr)audioProcessingModule networkControllerFactory:(std::unique_ptr) - networkControllerFactory; + networkControllerFactory + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing; - (instancetype) initWithEncoderFactory:(nullable id)encoderFactory diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index 17777f6d5d..81a43bcf16 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -24,6 +24,7 @@ NS_ASSUME_NONNULL_BEGIN @class RTC_OBJC_TYPE(RTCVideoSource); @class RTC_OBJC_TYPE(RTCVideoTrack); @class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions); +@class RTC_OBJC_TYPE(RTCAudioDeviceModule); @protocol RTC_OBJC_TYPE (RTCPeerConnectionDelegate); @protocol RTC_OBJC_TYPE @@ -34,6 +35,8 @@ NS_ASSUME_NONNULL_BEGIN (RTCSSLCertificateVerifier); @protocol RTC_OBJC_TYPE (RTCAudioDevice); +@protocol RTC_OBJC_TYPE +(RTCAudioProcessingModule); RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject @@ -52,6 +55,16 @@ RTC_OBJC_EXPORT decoderFactory:(nullable id)decoderFactory audioDevice:(nullable id)audioDevice; +/* Initialize object with bypass voice processing */ +- (instancetype) + initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing + encoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + audioProcessingModule: + (nullable id)audioProcessingModule; + +@property(nonatomic, readonly) RTCAudioDeviceModule *audioDeviceModule; + /** * Valid kind values are kRTCMediaStreamTrackKindAudio and * kRTCMediaStreamTrackKindVideo. diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index 64be41ae15..36011d045f 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -15,6 +15,9 @@ #import "RTCPeerConnectionFactoryOptions+Private.h" #import "RTCRtpCapabilities+Private.h" +#import "RTCAudioDeviceModule.h" +#import "RTCAudioDeviceModule+Private.h" + #import "RTCAudioSource+Private.h" #import "RTCAudioTrack+Private.h" #import "RTCMediaConstraints+Private.h" @@ -49,6 +52,9 @@ #include "sdk/objc/native/src/objc_video_decoder_factory.h" #include "sdk/objc/native/src/objc_video_encoder_factory.h" +#import "components/audio/RTCAudioProcessingModule.h" +#import "components/audio/RTCDefaultAudioProcessingModule+Private.h" + #if defined(WEBRTC_IOS) #import "sdk/objc/native/api/audio_device_module.h" #endif @@ -57,14 +63,18 @@ @implementation RTC_OBJC_TYPE (RTCPeerConnectionFactory) { std::unique_ptr _networkThread; std::unique_ptr _workerThread; std::unique_ptr _signalingThread; + rtc::scoped_refptr _nativeAudioDeviceModule; + RTCDefaultAudioProcessingModule *_defaultAudioProcessingModule; + BOOL _hasStartedAecDump; } @synthesize nativeFactory = _nativeFactory; +@synthesize audioDeviceModule = _audioDeviceModule; -- (rtc::scoped_refptr)audioDeviceModule { +- (rtc::scoped_refptr)createAudioDeviceModule:(BOOL)bypassVoiceProcessing { #if defined(WEBRTC_IOS) - return webrtc::CreateAudioDeviceModule(); + return webrtc::CreateAudioDeviceModule(bypassVoiceProcessing); #else return nullptr; #endif @@ -78,8 +88,9 @@ - (instancetype)init { RTCVideoEncoderFactoryH264) alloc] init]) nativeVideoDecoderFactory:webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE( RTCVideoDecoderFactoryH264) alloc] init]) - audioDeviceModule:[self audioDeviceModule].get() - audioProcessingModule:nullptr]; + audioDeviceModule:[self createAudioDeviceModule:NO].get() + audioProcessingModule:nullptr + bypassVoiceProcessing:NO]; } - (instancetype) @@ -107,14 +118,52 @@ - (instancetype)init { if (audioDevice) { audio_device_module = webrtc::CreateAudioDeviceModule(audioDevice); } else { - audio_device_module = [self audioDeviceModule]; + audio_device_module = [self createAudioDeviceModule:NO]; } return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) audioDeviceModule:audio_device_module.get() - audioProcessingModule:nullptr]; + audioProcessingModule:nullptr + bypassVoiceProcessing:NO]; +#endif +} + +- (instancetype) + initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing + encoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + audioProcessingModule: + (nullable id)audioProcessingModule { +#ifdef HAVE_NO_MEDIA + return [self initWithNoMedia]; +#else + std::unique_ptr native_encoder_factory; + std::unique_ptr native_decoder_factory; + if (encoderFactory) { + native_encoder_factory = webrtc::ObjCToNativeVideoEncoderFactory(encoderFactory); + } + if (decoderFactory) { + native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory); + } + rtc::scoped_refptr audio_device_module = [self createAudioDeviceModule:bypassVoiceProcessing]; + + if ([audioProcessingModule isKindOfClass:[RTCDefaultAudioProcessingModule class]]) { + _defaultAudioProcessingModule = (RTCDefaultAudioProcessingModule *)audioProcessingModule; + } else { + _defaultAudioProcessingModule = [[RTCDefaultAudioProcessingModule alloc] init]; + } + + NSLog(@"AudioProcessingModule: %@", _defaultAudioProcessingModule); + + return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() + nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() + nativeVideoEncoderFactory:std::move(native_encoder_factory) + nativeVideoDecoderFactory:std::move(native_decoder_factory) + audioDeviceModule:audio_device_module.get() + audioProcessingModule:_defaultAudioProcessingModule.nativeAudioProcessingModule + bypassVoiceProcessing:bypassVoiceProcessing]; #endif } @@ -163,14 +212,16 @@ - (instancetype)initWithNativeAudioEncoderFactory: (std::unique_ptr)videoDecoderFactory audioDeviceModule:(webrtc::AudioDeviceModule *)audioDeviceModule audioProcessingModule: - (rtc::scoped_refptr)audioProcessingModule { + (rtc::scoped_refptr)audioProcessingModule + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing { return [self initWithNativeAudioEncoderFactory:audioEncoderFactory nativeAudioDecoderFactory:audioDecoderFactory nativeVideoEncoderFactory:std::move(videoEncoderFactory) nativeVideoDecoderFactory:std::move(videoDecoderFactory) audioDeviceModule:audioDeviceModule audioProcessingModule:audioProcessingModule - networkControllerFactory:nullptr]; + networkControllerFactory:nullptr + bypassVoiceProcessing:NO]; } - (instancetype)initWithNativeAudioEncoderFactory: (rtc::scoped_refptr)audioEncoderFactory @@ -185,7 +236,8 @@ - (instancetype)initWithNativeAudioEncoderFactory: (rtc::scoped_refptr)audioProcessingModule networkControllerFactory: (std::unique_ptr) - networkControllerFactory { + networkControllerFactory + bypassVoiceProcessing:(BOOL)bypassVoiceProcessing { if (self = [self initNative]) { webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = _networkThread.get(); @@ -197,11 +249,22 @@ - (instancetype)initWithNativeAudioEncoderFactory: dependencies.trials = std::make_unique(); dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory(dependencies.trials.get()); - dependencies.adm = std::move(audioDeviceModule); + + // always create ADM on worker thread + _nativeAudioDeviceModule = _workerThread->BlockingCall([&dependencies, &bypassVoiceProcessing]() { + return webrtc::AudioDeviceModule::Create(webrtc::AudioDeviceModule::AudioLayer::kPlatformDefaultAudio, + dependencies.task_queue_factory.get(), + bypassVoiceProcessing == YES); + }); + + _audioDeviceModule = [[RTCAudioDeviceModule alloc] initWithNativeModule: _nativeAudioDeviceModule + workerThread: _workerThread.get()]; + dependencies.adm = _nativeAudioDeviceModule; dependencies.audio_encoder_factory = std::move(audioEncoderFactory); dependencies.audio_decoder_factory = std::move(audioDecoderFactory); dependencies.video_encoder_factory = std::move(videoEncoderFactory); dependencies.video_decoder_factory = std::move(videoDecoderFactory); + if (audioProcessingModule) { dependencies.audio_processing = std::move(audioProcessingModule); } else { diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm index 627909a0e3..0981fb3879 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactoryBuilder.mm @@ -39,7 +39,8 @@ + (RTCPeerConnectionFactoryBuilder *)builder { nativeVideoEncoderFactory:std::move(_videoEncoderFactory) nativeVideoDecoderFactory:std::move(_videoDecoderFactory) audioDeviceModule:_audioDeviceModule.get() - audioProcessingModule:_audioProcessingModule]; + audioProcessingModule:_audioProcessingModule + bypassVoiceProcessing:NO]; } - (void)setVideoEncoderFactory:(std::unique_ptr)videoEncoderFactory { diff --git a/sdk/objc/base/RTCAudioRenderer.h b/sdk/objc/base/RTCAudioRenderer.h new file mode 100644 index 0000000000..def20eac3c --- /dev/null +++ b/sdk/objc/base/RTCAudioRenderer.h @@ -0,0 +1,34 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#if TARGET_OS_IPHONE +#import +#endif + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioRenderer) + + - (void)renderSampleBuffer : (CMSampleBufferRef)sampleBuffer + NS_SWIFT_NAME(render(sampleBuffer:)); + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioBuffer+Private.h b/sdk/objc/components/audio/RTCAudioBuffer+Private.h new file mode 100644 index 0000000000..effd8bb429 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioBuffer+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioBuffer.h" + +#include "modules/audio_processing/audio_buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCAudioBuffer)() + +- (instancetype)initWithNativeType: (webrtc::AudioBuffer *) audioBuffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioBuffer.h b/sdk/objc/components/audio/RTCAudioBuffer.h new file mode 100644 index 0000000000..8bbd068657 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioBuffer.h @@ -0,0 +1,38 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioBuffer) : NSObject + +@property(nonatomic, readonly) size_t channels; +@property(nonatomic, readonly) size_t frames; +@property(nonatomic, readonly) size_t framesPerBand; +@property(nonatomic, readonly) size_t bands; + +// Returns pointer arrays. Index range from 0 to `frames`. +- (float* _Nonnull)rawBufferForChannel:(size_t)channel; + +// TODO: More convenience methods... + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioBuffer.mm b/sdk/objc/components/audio/RTCAudioBuffer.mm new file mode 100644 index 0000000000..e37ea344dd --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioBuffer.mm @@ -0,0 +1,55 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioBuffer.h" + +#include "modules/audio_processing/audio_buffer.h" + +@implementation RTC_OBJC_TYPE (RTCAudioBuffer) { + // Raw + webrtc::AudioBuffer *_audioBuffer; +} + +- (size_t)channels { + return _audioBuffer->num_channels(); +} + +- (size_t)frames { + return _audioBuffer->num_frames(); +} + +- (size_t)framesPerBand { + return _audioBuffer->num_frames_per_band(); +} + +- (size_t)bands { + return _audioBuffer->num_bands(); +} + +- (float *)rawBufferForChannel:(size_t)channel { + return _audioBuffer->channels()[channel]; +} + +#pragma mark - Private + +- (instancetype)initWithNativeType:(webrtc::AudioBuffer *)audioBuffer { + if (self = [super init]) { + _audioBuffer = audioBuffer; + } + return self; +} + +@end diff --git a/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h new file mode 100644 index 0000000000..a9dc3d8400 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter+Private.h @@ -0,0 +1,43 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioCustomProcessingAdapter.h" +#import "RTCAudioCustomProcessingDelegate.h" +#import "RTCMacros.h" + +#include "modules/audio_processing/include/audio_processing.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCAudioCustomProcessingAdapter () + +// Thread safe set/get with os_unfair_lock. +@property(nonatomic, weak, nullable) id + audioCustomProcessingDelegate; + +// Direct read access without lock. +@property(nonatomic, readonly, weak, nullable) id + rawAudioCustomProcessingDelegate; + +@property(nonatomic, readonly) std::unique_ptr + nativeAudioCustomProcessingModule; + +- (instancetype)initWithDelegate: + (nullable id)audioCustomProcessingDelegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h new file mode 100644 index 0000000000..24239eac2d --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.h @@ -0,0 +1,28 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCAudioCustomProcessingAdapter : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.mm b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.mm new file mode 100644 index 0000000000..c8d1dfe4f6 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioCustomProcessingAdapter.mm @@ -0,0 +1,139 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import + +#import "RTCAudioBuffer+Private.h" +#import "RTCAudioCustomProcessingAdapter+Private.h" +#import "RTCAudioCustomProcessingAdapter.h" + +#include "rtc_base/logging.h" + +namespace webrtc { + +class AudioCustomProcessingAdapter : public webrtc::CustomProcessing { + public: + bool is_initialized_; + int sample_rate_hz_; + int num_channels_; + + AudioCustomProcessingAdapter(RTCAudioCustomProcessingAdapter *adapter, os_unfair_lock *lock) { + RTC_LOG(LS_INFO) << "RTCAudioCustomProcessingAdapter.AudioCustomProcessingAdapter init"; + + adapter_ = adapter; + lock_ = lock; + is_initialized_ = false; + sample_rate_hz_ = 0; + num_channels_ = 0; + } + + ~AudioCustomProcessingAdapter() { + RTC_LOG(LS_INFO) << "RTCAudioCustomProcessingAdapter.AudioCustomProcessingAdapter dealloc"; + + os_unfair_lock_lock(lock_); + id delegate = adapter_.rawAudioCustomProcessingDelegate; + [delegate audioProcessingRelease]; + os_unfair_lock_unlock(lock_); + } + + void Initialize(int sample_rate_hz, int num_channels) override { + os_unfair_lock_lock(lock_); + id delegate = adapter_.rawAudioCustomProcessingDelegate; + [delegate audioProcessingInitializeWithSampleRate:sample_rate_hz channels:num_channels]; + is_initialized_ = true; + sample_rate_hz_ = sample_rate_hz; + num_channels_ = num_channels; + os_unfair_lock_unlock(lock_); + } + + void Process(AudioBuffer *audio_buffer) override { + bool is_locked = os_unfair_lock_trylock(lock_); + if (!is_locked) { + RTC_LOG(LS_INFO) << "RTCAudioCustomProcessingAdapter.AudioCustomProcessingAdapter Process " + "already locked, skipping..."; + + return; + } + id delegate = adapter_.rawAudioCustomProcessingDelegate; + if (delegate != nil) { + RTCAudioBuffer *audioBuffer = [[RTCAudioBuffer alloc] initWithNativeType:audio_buffer]; + [delegate audioProcessingProcess:audioBuffer]; + } + os_unfair_lock_unlock(lock_); + } + + std::string ToString() const override { return "AudioCustomProcessingAdapter"; } + + private: + __weak RTCAudioCustomProcessingAdapter *adapter_; + os_unfair_lock *lock_; +}; +} // namespace webrtc + +@implementation RTCAudioCustomProcessingAdapter { + webrtc::AudioCustomProcessingAdapter *_adapter; + os_unfair_lock _lock; +} + +@synthesize rawAudioCustomProcessingDelegate = _rawAudioCustomProcessingDelegate; + +- (instancetype)initWithDelegate: + (nullable id)audioCustomProcessingDelegate { + if (self = [super init]) { + _lock = OS_UNFAIR_LOCK_INIT; + _rawAudioCustomProcessingDelegate = audioCustomProcessingDelegate; + _adapter = new webrtc::AudioCustomProcessingAdapter(self, &_lock); + RTC_LOG(LS_INFO) << "RTCAudioCustomProcessingAdapter init"; + } + + return self; +} + +- (void)dealloc { + RTC_LOG(LS_INFO) << "RTCAudioCustomProcessingAdapter dealloc"; +} + +#pragma mark - Getter & Setter for audioCustomProcessingDelegate + +- (nullable id)audioCustomProcessingDelegate { + os_unfair_lock_lock(&_lock); + id delegate = _rawAudioCustomProcessingDelegate; + os_unfair_lock_unlock(&_lock); + return delegate; +} + +- (void)setAudioCustomProcessingDelegate:(nullable id)delegate { + os_unfair_lock_lock(&_lock); + if (_rawAudioCustomProcessingDelegate != nil && _adapter->is_initialized_) { + [_rawAudioCustomProcessingDelegate audioProcessingRelease]; + } + _rawAudioCustomProcessingDelegate = delegate; + if (_adapter->is_initialized_) { + [_rawAudioCustomProcessingDelegate + audioProcessingInitializeWithSampleRate:_adapter->sample_rate_hz_ + channels:_adapter->num_channels_]; + } + os_unfair_lock_unlock(&_lock); +} + +#pragma mark - Private + +- (std::unique_ptr)nativeAudioCustomProcessingModule { + return std::unique_ptr(_adapter); +} + +@end diff --git a/sdk/objc/components/audio/RTCAudioCustomProcessingDelegate.h b/sdk/objc/components/audio/RTCAudioCustomProcessingDelegate.h new file mode 100644 index 0000000000..6a2fec9433 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioCustomProcessingDelegate.h @@ -0,0 +1,52 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioBuffer); + +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE (RTCAudioCustomProcessingDelegate) + +/** +* (Re-)initialize the audio processor. +* This method can be invoked multiple times. +*/ +- (void)audioProcessingInitializeWithSampleRate : (size_t)sampleRateHz channels +: (size_t)channels NS_SWIFT_NAME(audioProcessingInitialize(sampleRate:channels:)); + +/** + * Process (read or write) the audio buffer. + * RTCAudioBuffer is a simple wrapper for webrtc::AudioBuffer and the valid scope is only inside + * this method. Do not retain it. + */ +- (void)audioProcessingProcess:(RTCAudioBuffer *)audioBuffer + NS_SWIFT_NAME(audioProcessingProcess(audioBuffer:)); + +// TOOD: +// virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting); + +/** + * Suggests releasing resources allocated by the audio processor. + */ +- (void)audioProcessingRelease; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioProcessingConfig+Private.h b/sdk/objc/components/audio/RTCAudioProcessingConfig+Private.h new file mode 100644 index 0000000000..ed565ee0aa --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioProcessingConfig+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioProcessingConfig.h" + +#include "modules/audio_processing/include/audio_processing.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCAudioProcessingConfig)() + +@property(nonatomic, readonly) webrtc::AudioProcessing::Config nativeAudioProcessingConfig; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioProcessingConfig.h b/sdk/objc/components/audio/RTCAudioProcessingConfig.h new file mode 100644 index 0000000000..3c7dce45f1 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioProcessingConfig.h @@ -0,0 +1,31 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCAudioProcessingConfig) : NSObject + +@property(nonatomic, assign) BOOL echoCancellerEnabled; +@property(nonatomic, assign) BOOL echoCancellerMobileMode; + + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioProcessingConfig.mm b/sdk/objc/components/audio/RTCAudioProcessingConfig.mm new file mode 100644 index 0000000000..ca40f16e17 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioProcessingConfig.mm @@ -0,0 +1,51 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCAudioProcessingConfig.h" + +#include "modules/audio_processing/include/audio_processing.h" + +@implementation RTC_OBJC_TYPE (RTCAudioProcessingConfig) { + webrtc::AudioProcessing::Config _config; +} + +// config.echo_canceller.enabled + +- (BOOL)echoCancellerEnabled { + return _config.echo_canceller.enabled; +} + +- (void)setEchoCancellerEnabled:(BOOL)value { + _config.echo_canceller.enabled = value; +} + +// config.echo_canceller.mobile_mode + +- (BOOL)echoCancellerMobileMode { + return _config.echo_canceller.mobile_mode; +} + +- (void)setEchoCancellerMobileMode:(BOOL)value { + _config.echo_canceller.mobile_mode = value; +} + +#pragma mark - Private + +- (webrtc::AudioProcessing::Config)nativeAudioProcessingConfig { + return _config; +} + +@end diff --git a/sdk/objc/components/audio/RTCAudioProcessingModule.h b/sdk/objc/components/audio/RTCAudioProcessingModule.h new file mode 100644 index 0000000000..7bf0402427 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioProcessingModule.h @@ -0,0 +1,33 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioProcessingConfig); + +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE (RTCAudioProcessingModule) + +- (void)applyConfig: (RTCAudioProcessingConfig *)config; + +// TODO: Implement... + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm index 4bf0a542e1..f7b45b5850 100644 --- a/sdk/objc/components/audio/RTCAudioSession+Configuration.mm +++ b/sdk/objc/components/audio/RTCAudioSession+Configuration.mm @@ -63,6 +63,33 @@ - (BOOL)setConfiguration:(RTC_OBJC_TYPE(RTCAudioSessionConfiguration) *)configur } } + if (self.mode != configuration.mode) { + NSError *modeError = nil; + if (![self setMode:configuration.mode error:&modeError]) { + RTCLogError(@"Failed to set mode to %@: %@", + self.mode, + modeError.localizedDescription); + error = modeError; + } else { + RTCLog(@"Set mode to: %@", configuration.mode); + } + } + + // Sometimes category options don't stick after setting mode. + if (self.categoryOptions != configuration.categoryOptions) { + NSError *categoryError = nil; + if (![self setCategory:configuration.category + withOptions:configuration.categoryOptions + error:&categoryError]) { + RTCLogError(@"Failed to set category options: %@", + categoryError.localizedDescription); + error = categoryError; + } else { + RTCLog(@"Set category options to: %ld", + (long)configuration.categoryOptions); + } + } + if (self.preferredSampleRate != configuration.sampleRate) { NSError *sampleRateError = nil; if (![self setPreferredSampleRate:configuration.sampleRate diff --git a/sdk/objc/components/audio/RTCAudioSession.mm b/sdk/objc/components/audio/RTCAudioSession.mm index 641d2ed8c7..083d4737c4 100644 --- a/sdk/objc/components/audio/RTCAudioSession.mm +++ b/sdk/objc/components/audio/RTCAudioSession.mm @@ -552,8 +552,7 @@ - (void)handleRouteChangeNotification:(NSNotification *)notification { RTCLog(@"Audio route changed: OldDeviceUnavailable"); break; case AVAudioSessionRouteChangeReasonCategoryChange: - RTCLog(@"Audio route changed: CategoryChange to :%@", - self.session.category); + RTCLog(@"Audio route changed: CategoryChange to :%@", self.session.category); break; case AVAudioSessionRouteChangeReasonOverride: RTCLog(@"Audio route changed: Override"); diff --git a/sdk/objc/components/audio/RTCAudioSessionConfiguration.m b/sdk/objc/components/audio/RTCAudioSessionConfiguration.m index 71b0c0cb3a..76e17e7afb 100644 --- a/sdk/objc/components/audio/RTCAudioSessionConfiguration.m +++ b/sdk/objc/components/audio/RTCAudioSessionConfiguration.m @@ -54,15 +54,17 @@ @implementation RTC_OBJC_TYPE (RTCAudioSessionConfiguration) - (instancetype)init { if (self = [super init]) { + // Use AVAudioSession values for default + AVAudioSession *session = [AVAudioSession sharedInstance]; // Use a category which supports simultaneous recording and playback. // By default, using this category implies that our app’s audio is // nonmixable, hence activating the session will interrupt any other // audio sessions which are also nonmixable. - _category = AVAudioSessionCategoryPlayAndRecord; - _categoryOptions = AVAudioSessionCategoryOptionAllowBluetooth; + _category = session.category; + _categoryOptions = session.categoryOptions; // Specify mode for two-way voice communication (e.g. VoIP). - _mode = AVAudioSessionModeVoiceChat; + _mode = session.mode; // Use best sample rate and buffer duration if the CPU has more than one // core. diff --git a/sdk/objc/components/audio/RTCDefaultAudioProcessingModule+Private.h b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule+Private.h new file mode 100644 index 0000000000..4f8551e372 --- /dev/null +++ b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule+Private.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDefaultAudioProcessingModule.h" + +#include "modules/audio_processing/include/audio_processing.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTC_OBJC_TYPE (RTCDefaultAudioProcessingModule)() + +@property(nonatomic, readonly) rtc::scoped_refptr nativeAudioProcessingModule; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.h b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.h new file mode 100644 index 0000000000..917d584d48 --- /dev/null +++ b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.h @@ -0,0 +1,47 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "RTCAudioProcessingModule.h" +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTC_OBJC_TYPE(RTCAudioProcessingConfig); +@protocol RTC_OBJC_TYPE +(RTCAudioCustomProcessingDelegate); + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCDefaultAudioProcessingModule) : NSObject + +- (instancetype)initWithConfig: (nullable RTCAudioProcessingConfig *)config + capturePostProcessingDelegate: (nullable id)capturePostProcessingDelegate + renderPreProcessingDelegate: (nullable id)renderPreProcessingDelegate + NS_SWIFT_NAME(init(config:capturePostProcessingDelegate:renderPreProcessingDelegate:)) NS_DESIGNATED_INITIALIZER; + +- (void)applyConfig:(RTCAudioProcessingConfig *)config; + +// Dynamically update delegates at runtime + +@property(nonatomic, weak, nullable) id + capturePostProcessingDelegate; +@property(nonatomic, weak, nullable) id + renderPreProcessingDelegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.mm b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.mm new file mode 100644 index 0000000000..3875dec533 --- /dev/null +++ b/sdk/objc/components/audio/RTCDefaultAudioProcessingModule.mm @@ -0,0 +1,96 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "RTCDefaultAudioProcessingModule.h" +#import "RTCAudioCustomProcessingAdapter+Private.h" +#import "RTCAudioProcessingConfig+Private.h" + +#include "modules/audio_processing/include/audio_processing.h" + +@implementation RTC_OBJC_TYPE (RTCDefaultAudioProcessingModule) { + rtc::scoped_refptr _nativeAudioProcessingModule; + // Custom processing adapters... + RTCAudioCustomProcessingAdapter *_capturePostProcessingAdapter; + RTCAudioCustomProcessingAdapter *_renderPreProcessingAdapter; +} + +- (instancetype)init { + return [self initWithConfig:nil + capturePostProcessingDelegate:nil + renderPreProcessingDelegate:nil]; +} + +- (instancetype)initWithConfig:(nullable RTCAudioProcessingConfig *)config + capturePostProcessingDelegate: + (nullable id)capturePostProcessingDelegate + renderPreProcessingDelegate:(nullable id) + renderPreProcessingDelegate { + if (self = [super init]) { + webrtc::AudioProcessingBuilder builder = webrtc::AudioProcessingBuilder(); + + // TODO: Custom Config... + + if (config != nil) { + builder.SetConfig(config.nativeAudioProcessingConfig); + } + + _capturePostProcessingAdapter = + [[RTCAudioCustomProcessingAdapter alloc] initWithDelegate:capturePostProcessingDelegate]; + builder.SetCapturePostProcessing( + _capturePostProcessingAdapter.nativeAudioCustomProcessingModule); + + _renderPreProcessingAdapter = + [[RTCAudioCustomProcessingAdapter alloc] initWithDelegate:renderPreProcessingDelegate]; + builder.SetRenderPreProcessing(_renderPreProcessingAdapter.nativeAudioCustomProcessingModule); + + _nativeAudioProcessingModule = builder.Create(); + } + return self; +} + +#pragma mark - Getter & Setters for delegates + +- (nullable id)capturePostProcessingDelegate { + return _capturePostProcessingAdapter.audioCustomProcessingDelegate; +} + +- (void)setCapturePostProcessingDelegate: + (nullable id)delegate { + _capturePostProcessingAdapter.audioCustomProcessingDelegate = delegate; +} + +- (nullable id)renderPreProcessingDelegate { + return _renderPreProcessingAdapter.audioCustomProcessingDelegate; +} + +- (void)setRenderPreProcessingDelegate: + (nullable id)delegate { + _renderPreProcessingAdapter.audioCustomProcessingDelegate = delegate; +} + +#pragma mark - RTCAudioProcessingModule protocol + +- (void)applyConfig:(RTCAudioProcessingConfig *)config { + _nativeAudioProcessingModule->ApplyConfig(config.nativeAudioProcessingConfig); +} + +#pragma mark - Private + +- (rtc::scoped_refptr)nativeAudioProcessingModule { + return _nativeAudioProcessingModule; +} + +@end diff --git a/sdk/objc/native/api/audio_device_module.mm b/sdk/objc/native/api/audio_device_module.mm index 4e7b681e69..ada25bd9ee 100644 --- a/sdk/objc/native/api/audio_device_module.mm +++ b/sdk/objc/native/api/audio_device_module.mm @@ -13,7 +13,11 @@ #include "api/make_ref_counted.h" #include "rtc_base/logging.h" +#if defined(WEBRTC_IOS) #include "sdk/objc/native/src/audio/audio_device_module_ios.h" +#endif + +#include "modules/audio_device/include/audio_device.h" namespace webrtc { diff --git a/sdk/objc/native/src/audio/audio_device_ios.h b/sdk/objc/native/src/audio/audio_device_ios.h index 4ef4d0b5df..877cb8bbf9 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.h +++ b/sdk/objc/native/src/audio/audio_device_ios.h @@ -172,6 +172,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric, void HandlePlayoutGlitchDetected(); void HandleOutputVolumeChange(); + bool RestartAudioUnit(bool enable_input); + // Uses current `playout_parameters_` and `record_parameters_` to inform the // audio device buffer (ADB) about our internal audio parameters. void UpdateAudioDeviceBuffer(); @@ -200,7 +202,7 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // Activates our audio session, creates and initializes the voice-processing // audio unit and verifies that we got the preferred native audio parameters. - bool InitPlayOrRecord(); + bool InitPlayOrRecord(bool enable_input); // Closes and deletes the voice-processing I/O unit. void ShutdownPlayOrRecord(); @@ -260,19 +262,19 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // will be changed dynamically to account for this behavior. rtc::BufferT record_audio_buffer_; + bool recording_is_initialized_; + // Set to 1 when recording is active and 0 otherwise. std::atomic recording_; + bool playout_is_initialized_; + // Set to 1 when playout is active and 0 otherwise. std::atomic playing_; // Set to true after successful call to Init(), false otherwise. bool initialized_ RTC_GUARDED_BY(thread_); - // Set to true after successful call to InitRecording() or InitPlayout(), - // false otherwise. - bool audio_is_initialized_; - // Set to true if audio session is interrupted, false otherwise. bool is_interrupted_; diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index 78420ec232..ed7260489a 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -62,6 +62,16 @@ const UInt16 kFixedPlayoutDelayEstimate = 30; const UInt16 kFixedRecordDelayEstimate = 30; +enum AudioDeviceMessageType : uint32_t { + kMessageTypeInterruptionBegin, + kMessageTypeInterruptionEnd, + kMessageTypeValidRouteChange, + kMessageTypeCanPlayOrRecordChange, + kMessageTypePlayoutGlitchDetected, + kMessageOutputVolumeChange, + kMessageTypeAudioWillRecord, +}; + using ios::CheckAndLogError; #if !defined(NDEBUG) @@ -95,10 +105,11 @@ static void LogDeviceInfo() { : bypass_voice_processing_(bypass_voice_processing), audio_device_buffer_(nullptr), audio_unit_(nullptr), + recording_is_initialized_(false), recording_(0), + playout_is_initialized_(false), playing_(0), initialized_(false), - audio_is_initialized_(false), is_interrupted_(false), has_configured_session_(false), num_detected_playout_glitches_(0), @@ -180,48 +191,57 @@ static void LogDeviceInfo() { LOGI() << "InitPlayout"; RTC_DCHECK_RUN_ON(thread_); RTC_DCHECK(initialized_); - RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!playout_is_initialized_); RTC_DCHECK(!playing_.load()); - if (!audio_is_initialized_) { - if (!InitPlayOrRecord()) { + if (!recording_is_initialized_) { + // recording not initialized yet, init with no input + if (!InitPlayOrRecord(false)) { RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitPlayout!"; return -1; } } - audio_is_initialized_ = true; + + playout_is_initialized_ = true; + return 0; } bool AudioDeviceIOS::PlayoutIsInitialized() const { RTC_DCHECK_RUN_ON(thread_); - return audio_is_initialized_; + return playout_is_initialized_; } bool AudioDeviceIOS::RecordingIsInitialized() const { RTC_DCHECK_RUN_ON(thread_); - return audio_is_initialized_; + return recording_is_initialized_; } int32_t AudioDeviceIOS::InitRecording() { LOGI() << "InitRecording"; RTC_DCHECK_RUN_ON(thread_); RTC_DCHECK(initialized_); - RTC_DCHECK(!audio_is_initialized_); + RTC_DCHECK(!recording_is_initialized_); RTC_DCHECK(!recording_.load()); - if (!audio_is_initialized_) { - if (!InitPlayOrRecord()) { + if (!playout_is_initialized_) { + // playout not initialized yet, init with input + if (!InitPlayOrRecord(true)) { RTC_LOG_F(LS_ERROR) << "InitPlayOrRecord failed for InitRecording!"; return -1; } + } else { + // playout already initialized, restart audio unit with input + RestartAudioUnit(true); } - audio_is_initialized_ = true; + + recording_is_initialized_ = true; + return 0; } int32_t AudioDeviceIOS::StartPlayout() { LOGI() << "StartPlayout"; RTC_DCHECK_RUN_ON(thread_); - RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(playout_is_initialized_); RTC_DCHECK(!playing_.load()); RTC_DCHECK(audio_unit_); if (fine_audio_buffer_) { @@ -246,14 +266,16 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StopPlayout() { LOGI() << "StopPlayout"; RTC_DCHECK_RUN_ON(thread_); - if (!audio_is_initialized_ || !playing_.load()) { + if (!playout_is_initialized_ || !playing_.load()) { return 0; } if (!recording_.load()) { ShutdownPlayOrRecord(); - audio_is_initialized_ = false; + + recording_is_initialized_ = false; } playing_.store(0, std::memory_order_release); + playout_is_initialized_ = false; // Derive average number of calls to OnGetPlayoutData() between detected // audio glitches and add the result to a histogram. @@ -277,7 +299,7 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StartRecording() { LOGI() << "StartRecording"; RTC_DCHECK_RUN_ON(thread_); - RTC_DCHECK(audio_is_initialized_); + RTC_DCHECK(recording_is_initialized_); RTC_DCHECK(!recording_.load()); RTC_DCHECK(audio_unit_); if (fine_audio_buffer_) { @@ -300,14 +322,19 @@ static void LogDeviceInfo() { int32_t AudioDeviceIOS::StopRecording() { LOGI() << "StopRecording"; RTC_DCHECK_RUN_ON(thread_); - if (!audio_is_initialized_ || !recording_.load()) { + if (!recording_is_initialized_ || !recording_.load()) { return 0; } if (!playing_.load()) { ShutdownPlayOrRecord(); - audio_is_initialized_ = false; + + playout_is_initialized_ = false; + } else if (playout_is_initialized_) { + // restart audio unit with no input + RestartAudioUnit(false); } recording_.store(0, std::memory_order_release); + recording_is_initialized_ = false; return 0; } @@ -455,7 +482,7 @@ static void LogDeviceInfo() { // Exclude extreme delta values since they do most likely not correspond // to a real glitch. Instead, the most probable cause is that a headset // has been plugged in or out. There are more direct ways to detect - // audio device changes (see HandleValidRouteChange()) but experiments + // audio device changes (see ValidRouteChange()) but experiments // show that using it leads to more complex implementations. // TODO(henrika): more tests might be needed to come up with an even // better upper limit. @@ -589,7 +616,7 @@ static void LogDeviceInfo() { SetupAudioBuffersForActiveAudioSession(); // Initialize the audio unit again with the new sample rate. - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), recording_is_initialized_)) { RTCLogError(@"Failed to initialize the audio unit with sample rate: %d", playout_parameters_.sample_rate()); return; @@ -643,6 +670,46 @@ static void LogDeviceInfo() { last_output_volume_change_time_ = rtc::TimeMillis(); } +bool AudioDeviceIOS::RestartAudioUnit(bool enable_input) { + RTC_DCHECK_RUN_ON(&io_thread_checker_); + + LOGI() << "RestartAudioUnit"; + + // If we don't have an audio unit yet, or the audio unit is uninitialized, + // there is no work to do. + if (!audio_unit_ || audio_unit_->GetState() < VoiceProcessingAudioUnit::kInitialized) { + return false; + } + + bool restart_audio_unit = false; + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kStarted) { + audio_unit_->Stop(); + PrepareForNewStart(); + restart_audio_unit = true; + } + + if (audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) { + audio_unit_->Uninitialize(); + } + + // Initialize the audio unit again with the same sample rate. + const double sample_rate = playout_parameters_.sample_rate(); + + if (!audio_unit_->Initialize(sample_rate, enable_input)) { + RTCLogError(@"Failed to initialize the audio unit with sample rate: %f", sample_rate); + return false; + } + + // Restart the audio unit if it was already running. + if (restart_audio_unit && !audio_unit_->Start()) { + RTCLogError(@"Failed to start audio unit with sample rate: %f", sample_rate); + return false; + } + + LOGI() << "Successfully enabled audio unit for recording."; + return true; +} + void AudioDeviceIOS::UpdateAudioDeviceBuffer() { LOGI() << "UpdateAudioDevicebuffer"; // AttachAudioBuffer() is called at construction by the main class but check @@ -736,7 +803,7 @@ static void LogDeviceInfo() { // If we're not initialized we don't need to do anything. Audio unit will // be initialized on initialization. - if (!audio_is_initialized_) return; + if (!playout_is_initialized_ && !recording_is_initialized_) return; // If we're initialized, we must have an audio unit. RTC_DCHECK(audio_unit_); @@ -774,7 +841,7 @@ static void LogDeviceInfo() { RTCLog(@"Initializing audio unit for UpdateAudioUnit"); ConfigureAudioSession(); SetupAudioBuffersForActiveAudioSession(); - if (!audio_unit_->Initialize(playout_parameters_.sample_rate())) { + if (!audio_unit_->Initialize(playout_parameters_.sample_rate(), recording_is_initialized_)) { RTCLogError(@"Failed to initialize audio unit."); return; } @@ -864,7 +931,7 @@ static void LogDeviceInfo() { RTCLog(@"Unconfigured audio session."); } -bool AudioDeviceIOS::InitPlayOrRecord() { +bool AudioDeviceIOS::InitPlayOrRecord(bool enable_input) { LOGI() << "InitPlayOrRecord"; RTC_DCHECK_RUN_ON(thread_); @@ -900,7 +967,7 @@ static void LogDeviceInfo() { return false; } SetupAudioBuffersForActiveAudioSession(); - audio_unit_->Initialize(playout_parameters_.sample_rate()); + audio_unit_->Initialize(playout_parameters_.sample_rate(), enable_input); } // Release the lock. diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.h b/sdk/objc/native/src/audio/audio_device_module_ios.h index 189d7e6c9c..2f9b95a0a8 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.h +++ b/sdk/objc/native/src/audio/audio_device_module_ios.h @@ -129,6 +129,9 @@ class AudioDeviceModuleIOS : public AudioDeviceModule { int GetPlayoutAudioParameters(AudioParameters* params) const override; int GetRecordAudioParameters(AudioParameters* params) const override; #endif // WEBRTC_IOS + + int32_t SetAudioDeviceSink(AudioDeviceSink* sink) const override; + private: const bool bypass_voice_processing_; bool initialized_ = false; diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.mm b/sdk/objc/native/src/audio/audio_device_module_ios.mm index 5effef3abd..5f93a06ee8 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_module_ios.mm @@ -665,5 +665,11 @@ return r; } #endif // WEBRTC_IOS + + int32_t AudioDeviceModuleIOS::SetAudioDeviceSink(AudioDeviceSink* sink) const { + // not implemented + RTC_LOG(LS_WARNING) << __FUNCTION__ << "(" << sink << ") Not implemented"; + return -1; + } } } diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.h b/sdk/objc/native/src/audio/voice_processing_audio_unit.h index ed9dd98568..b474cda104 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.h +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.h @@ -75,7 +75,7 @@ class VoiceProcessingAudioUnit { VoiceProcessingAudioUnit::State GetState() const; // Initializes the underlying audio unit with the given sample rate. - bool Initialize(Float64 sample_rate); + bool Initialize(Float64 sample_rate, bool enable_input); // Starts the underlying audio unit. OSStatus Start(); diff --git a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm index 3905b6857a..b3daacb334 100644 --- a/sdk/objc/native/src/audio/voice_processing_audio_unit.mm +++ b/sdk/objc/native/src/audio/voice_processing_audio_unit.mm @@ -111,19 +111,6 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return false; } - // Enable input on the input scope of the input element. - UInt32 enable_input = 1; - result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enable_input, - sizeof(enable_input)); - if (result != noErr) { - DisposeAudioUnit(); - RTCLogError(@"Failed to enable input on input scope of input element. " - "Error=%ld.", - (long)result); - return false; - } - // Enable output on the output scope of the output element. UInt32 enable_output = 1; result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, @@ -193,7 +180,7 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { return state_; } -bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate) { +bool VoiceProcessingAudioUnit::Initialize(Float64 sample_rate, bool enable_input) { RTC_DCHECK_GE(state_, kUninitialized); RTCLog(@"Initializing audio unit with sample rate: %f", sample_rate); @@ -204,6 +191,19 @@ static OSStatus GetAGCState(AudioUnit audio_unit, UInt32* enabled) { LogStreamDescription(format); #endif + UInt32 _enable_input = enable_input ? 1 : 0; + RTCLog(@"Initializing AudioUnit, _enable_input=%d", (int) _enable_input); + result = AudioUnitSetProperty(vpio_unit_, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &_enable_input, + sizeof(_enable_input)); + if (result != noErr) { + DisposeAudioUnit(); + RTCLogError(@"Failed to enable input on input scope of input element. " + "Error=%ld.", + (long)result); + return false; + } + // Set the format on the output scope of the input element/bus. result = AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,