diff --git a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift index bb453e293..8b21f2638 100644 --- a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift +++ b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift @@ -40,9 +40,16 @@ public protocol AudioCustomProcessingDelegate { } class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate { - // + // MARK: - Public + public var target: AudioCustomProcessingDelegate? { _state.target } + // MARK: - Internal + + let audioRenderers = MulticastDelegate(label: "AudioRenderer") + + // MARK: - Private + private struct State { weak var target: AudioCustomProcessingDelegate? } @@ -62,7 +69,13 @@ class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessing } func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) { - target?.audioProcessingProcess(audioBuffer: LKAudioBuffer(audioBuffer: audioBuffer)) + let lkAudioBuffer = LKAudioBuffer(audioBuffer: audioBuffer) + target?.audioProcessingProcess(audioBuffer: lkAudioBuffer) + + // Convert to pcmBuffer and notify only if an audioRenderer is added. + if audioRenderers.isDelegatesNotEmpty, let pcmBuffer = lkAudioBuffer.toAVAudioPCMBuffer() { + audioRenderers.notify { $0.render?(pcmBuffer: pcmBuffer) } + } } func audioProcessingRelease() { diff --git a/Sources/LiveKit/Protocols/AudioRenderer.swift b/Sources/LiveKit/Protocols/AudioRenderer.swift index 17db54def..38606011f 100644 --- a/Sources/LiveKit/Protocols/AudioRenderer.swift +++ b/Sources/LiveKit/Protocols/AudioRenderer.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import AVFoundation import CoreMedia #if swift(>=5.9) @@ -25,7 +26,11 @@ internal import LiveKitWebRTC @objc public protocol AudioRenderer { /// CMSampleBuffer for this track. + @objc optional func render(sampleBuffer: CMSampleBuffer) + + @objc optional + func render(pcmBuffer: AVAudioPCMBuffer) } class AudioRendererAdapter: NSObject, LKRTCAudioRenderer { @@ -36,7 +41,7 @@ class AudioRendererAdapter: NSObject, LKRTCAudioRenderer { } func render(sampleBuffer: CMSampleBuffer) { - target?.render(sampleBuffer: sampleBuffer) + target?.render?(sampleBuffer: sampleBuffer) } // Proxy the equality operators diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 80aa9ab82..944c505e4 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -313,3 +313,30 @@ public class AudioManager: Loggable { } #endif } + +public extension AudioManager { + /// Add an ``AudioRenderer`` to receive pcm buffers from local input (mic). + /// Only ``AudioRenderer/render(pcmBuffer:)`` will be called. + /// Usage: `AudioManager.shared.add(localAudioRenderer: localRenderer)` + func add(localAudioRenderer delegate: AudioRenderer) { + capturePostProcessingDelegateAdapter.audioRenderers.add(delegate: delegate) + } + + func remove(localAudioRenderer delegate: AudioRenderer) { + capturePostProcessingDelegateAdapter.audioRenderers.remove(delegate: delegate) + } +} + +public extension AudioManager { + /// Add an ``AudioRenderer`` to receive pcm buffers from combined remote audio. + /// Only ``AudioRenderer/render(pcmBuffer:)`` will be called. + /// To receive buffer for individual tracks, use ``RemoteAudioTrack/add(audioRenderer:)`` instead. + /// Usage: `AudioManager.shared.add(remoteAudioRenderer: localRenderer)` + func add(remoteAudioRenderer delegate: AudioRenderer) { + renderPreProcessingDelegateAdapter.audioRenderers.add(delegate: delegate) + } + + func remove(remoteAudioRenderer delegate: AudioRenderer) { + renderPreProcessingDelegateAdapter.audioRenderers.remove(delegate: delegate) + } +} diff --git a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift index d623aff18..2f04346bb 100644 --- a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift +++ b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift @@ -95,7 +95,7 @@ public class RemoteAudioTrack: Track, RemoteTrack, AudioTrack { extension RemoteAudioTrack: AudioRenderer { public func render(sampleBuffer: CMSampleBuffer) { _rendererState.audioRenderers.notify { audioRenderer in - audioRenderer.render(sampleBuffer: sampleBuffer) + audioRenderer.render?(sampleBuffer: sampleBuffer) } } }