From 198e96878e41898c942a2d7bc542a72c8b4b58bb Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:28:28 +0800 Subject: [PATCH 1/7] Signal --- Sources/LiveKit/Core/SignalClient.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Sources/LiveKit/Core/SignalClient.swift b/Sources/LiveKit/Core/SignalClient.swift index 0eef4fde2..59e08ccb7 100644 --- a/Sources/LiveKit/Core/SignalClient.swift +++ b/Sources/LiveKit/Core/SignalClient.swift @@ -510,6 +510,17 @@ extension SignalClient { try await _sendRequest(r) } + func sendUpdateLocalAudioTrack(sid _: String, features: [Livekit_AudioTrackFeature]) async throws { + let r = Livekit_SignalRequest.with { + $0.updateAudioTrack = Livekit_UpdateLocalAudioTrack.with { + $0.trackSid = "" + $0.features = features + } + } + + try await _sendRequest(r) + } + func sendSyncState(answer: Livekit_SessionDescription?, offer: Livekit_SessionDescription?, subscription: Livekit_UpdateSubscription, From 5ad1fe3178143b1dce87d7de9f890dde16ac5762 Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 5 Aug 2024 22:02:14 +0800 Subject: [PATCH 2/7] toFeatures --- .../LiveKit/Types/Options/AudioCaptureOptions.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift index 3a5f2d52a..b4e7e65d3 100644 --- a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift @@ -83,3 +83,14 @@ public class AudioCaptureOptions: NSObject, CaptureOptions { return hasher.finalize() } } + +// Internal +extension AudioCaptureOptions { + func toFeatures() -> Set { + Set([ + echoCancellation ? .tfEchoCancellation : nil, + noiseSuppression ? .tfNoiseSuppression : nil, + autoGainControl ? .tfAutoGainControl : nil, + ].compactMap { $0 }) + } +} From 04c469bb69ec209b9870b1f52ad07de53d71a300 Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 Aug 2024 12:11:57 +0800 Subject: [PATCH 3/7] Implement --- Sources/LiveKit/Core/SignalClient.swift | 6 +-- Sources/LiveKit/Track/AudioManager.swift | 33 +++++++++++++-- .../LiveKit/Track/Local/LocalAudioTrack.swift | 12 +++++- .../LocalTrackPublication.swift | 41 +++++++++++++++++++ .../TrackPublications/TrackPublication.swift | 2 + .../Types/Options/AudioPublishOptions.swift | 9 ++++ 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/Sources/LiveKit/Core/SignalClient.swift b/Sources/LiveKit/Core/SignalClient.swift index 59e08ccb7..73b57b368 100644 --- a/Sources/LiveKit/Core/SignalClient.swift +++ b/Sources/LiveKit/Core/SignalClient.swift @@ -510,11 +510,11 @@ extension SignalClient { try await _sendRequest(r) } - func sendUpdateLocalAudioTrack(sid _: String, features: [Livekit_AudioTrackFeature]) async throws { + func sendUpdateLocalAudioTrack(trackSid: Track.Sid, features: Set) async throws { let r = Livekit_SignalRequest.with { $0.updateAudioTrack = Livekit_UpdateLocalAudioTrack.with { - $0.trackSid = "" - $0.features = features + $0.trackSid = trackSid.stringValue + $0.features = Array(features) } } diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 6ec30b990..cb987b495 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -16,6 +16,7 @@ import Accelerate import AVFoundation +import Combine #if swift(>=5.9) internal import LiveKitWebRTC @@ -58,16 +59,35 @@ public class LKAudioBuffer: NSObject { @objc public protocol AudioCustomProcessingDelegate { + @objc optional + var audioProcessingName: String { get } + + @objc func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) + + @objc func audioProcessingProcess(audioBuffer: LKAudioBuffer) + + @objc func audioProcessingRelease() } class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate { - weak var target: AudioCustomProcessingDelegate? + // + public var target: AudioCustomProcessingDelegate? { _state.target } + + private struct State { + weak var target: AudioCustomProcessingDelegate? + } + + private var _state: StateSync init(target: AudioCustomProcessingDelegate? = nil) { - self.target = target + _state = StateSync(State(target: target)) + } + + public func set(target: AudioCustomProcessingDelegate?) { + _state.mutate { $0.target = target } } func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) { @@ -173,14 +193,19 @@ public class AudioManager: Loggable { return adapter }() + let capturePostProcessingDelegateSubject = CurrentValueSubject(nil) + public var capturePostProcessingDelegate: AudioCustomProcessingDelegate? { get { capturePostProcessingDelegateAdapter.target } - set { capturePostProcessingDelegateAdapter.target = newValue } + set { + capturePostProcessingDelegateAdapter.set(target: newValue) + capturePostProcessingDelegateSubject.send(newValue) + } } public var renderPreProcessingDelegate: AudioCustomProcessingDelegate? { get { renderPreProcessingDelegateAdapter.target } - set { renderPreProcessingDelegateAdapter.target = newValue } + set { renderPreProcessingDelegateAdapter.set(target: newValue) } } // MARK: - AudioDeviceModule diff --git a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift index 2f9c9ea19..1c419b639 100644 --- a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift +++ b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import Combine import Foundation #if swift(>=5.9) @@ -24,11 +25,17 @@ internal import LiveKitWebRTC @objc public class LocalAudioTrack: Track, LocalTrack, AudioTrack { + /// ``AudioCaptureOptions`` used to create this track. + let captureOptions: AudioCaptureOptions + init(name: String, source: Track.Source, track: LKRTCMediaStreamTrack, - reportStatistics: Bool) + reportStatistics: Bool, + captureOptions: AudioCaptureOptions) { + self.captureOptions = captureOptions + super.init(name: name, kind: .audio, source: source, @@ -62,7 +69,8 @@ public class LocalAudioTrack: Track, LocalTrack, AudioTrack { return LocalAudioTrack(name: name, source: .microphone, track: rtcTrack, - reportStatistics: reportStatistics) + reportStatistics: reportStatistics, + captureOptions: options) } @discardableResult diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index 4fbc052b9..ec49ba005 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import Combine import Foundation @objc @@ -26,6 +27,19 @@ public class LocalTrackPublication: TrackPublication { // MARK: - Private + private var _cancellable: AnyCancellable? + + override init(info: Livekit_TrackInfo, participant: Participant) { + super.init(info: info, participant: participant) + + // Watch audio manager processor changes. + _cancellable = AudioManager.shared.capturePostProcessingDelegateSubject.sink { [weak self] _ in + self?.recomputeAudioTrackFeatures() + } + } + + // MARK: - Private + private let _debounce = Debounce(delay: 0.1) public func mute() async throws { @@ -57,6 +71,8 @@ public class LocalTrackPublication: TrackPublication { newLocalVideoTrack.capturer.add(delegate: self) } + recomputeAudioTrackFeatures() + return oldValue } @@ -92,6 +108,31 @@ extension LocalTrackPublication: VideoCapturerDelegate { } extension LocalTrackPublication { + func recomputeAudioTrackFeatures() { + // ... + guard let audioTrack = track as? LocalAudioTrack else { return } + + print("recomputeAudioTrackFeatures: \(String(describing: track))") + + var features = audioTrack.captureOptions.toFeatures() + + // Check if Krisp is enabled. + if let processingDelegate = AudioManager.shared.capturePostProcessingDelegate, + processingDelegate.audioProcessingName == "krisp_noise_cancellation" + { + features.insert(.tfEnhancedNoiseCancellation) + } + + log("Sending audio track features: \(features)") + + Task.detached { [features] in + let participant = try await self.requireParticipant() + let room = try participant.requireRoom() + try await room.signalClient.sendUpdateLocalAudioTrack(trackSid: self.sid, + features: features) + } + } + func recomputeSenderParameters() { guard let track = track as? LocalVideoTrack, let sender = track._state.rtpSender else { return } diff --git a/Sources/LiveKit/TrackPublications/TrackPublication.swift b/Sources/LiveKit/TrackPublications/TrackPublication.swift index d1cce545f..f2addd332 100644 --- a/Sources/LiveKit/TrackPublications/TrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/TrackPublication.swift @@ -90,6 +90,8 @@ public class TrackPublication: NSObject, ObservableObject, Loggable { var encryptionType: EncryptionType = .none var latestInfo: Livekit_TrackInfo? + + var audioTrackFeatures: Set? } let _state: StateSync diff --git a/Sources/LiveKit/Types/Options/AudioPublishOptions.swift b/Sources/LiveKit/Types/Options/AudioPublishOptions.swift index 49cbd8b2d..fa140e7bd 100644 --- a/Sources/LiveKit/Types/Options/AudioPublishOptions.swift +++ b/Sources/LiveKit/Types/Options/AudioPublishOptions.swift @@ -61,3 +61,12 @@ public class AudioPublishOptions: NSObject, TrackPublishOptions { return hasher.finalize() } } + +// Internal +extension AudioPublishOptions { + func toFeatures() -> Set { + Set([ + !dtx ? .tfNoDtx : nil, + ].compactMap { $0 }) + } +} From 07f63d794c2b4b585e1048067cdf0ad7b2b7f0b0 Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:45:29 +0800 Subject: [PATCH 4/7] Only send if updated --- .../LocalTrackPublication.swift | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index ec49ba005..a958af417 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -34,7 +34,7 @@ public class LocalTrackPublication: TrackPublication { // Watch audio manager processor changes. _cancellable = AudioManager.shared.capturePostProcessingDelegateSubject.sink { [weak self] _ in - self?.recomputeAudioTrackFeatures() + self?.sendAudioTrackFeatures() } } @@ -71,7 +71,7 @@ public class LocalTrackPublication: TrackPublication { newLocalVideoTrack.capturer.add(delegate: self) } - recomputeAudioTrackFeatures() + sendAudioTrackFeatures() return oldValue } @@ -108,28 +108,34 @@ extension LocalTrackPublication: VideoCapturerDelegate { } extension LocalTrackPublication { - func recomputeAudioTrackFeatures() { - // ... + func sendAudioTrackFeatures() { + // Only proceed if audio track. guard let audioTrack = track as? LocalAudioTrack else { return } - print("recomputeAudioTrackFeatures: \(String(describing: track))") - - var features = audioTrack.captureOptions.toFeatures() + var newFeatures = audioTrack.captureOptions.toFeatures() // Check if Krisp is enabled. if let processingDelegate = AudioManager.shared.capturePostProcessingDelegate, processingDelegate.audioProcessingName == "krisp_noise_cancellation" { - features.insert(.tfEnhancedNoiseCancellation) + newFeatures.insert(.tfEnhancedNoiseCancellation) } - log("Sending audio track features: \(features)") + let didUpdateFeatures = _state.mutate { + let oldFeatures = $0.audioTrackFeatures + $0.audioTrackFeatures = newFeatures + return oldFeatures != newFeatures + } - Task.detached { [features] in - let participant = try await self.requireParticipant() - let room = try participant.requireRoom() - try await room.signalClient.sendUpdateLocalAudioTrack(trackSid: self.sid, - features: features) + if didUpdateFeatures { + log("Sending audio track features: \(newFeatures)") + // Send if features updated. + Task.detached { [newFeatures] in + let participant = try await self.requireParticipant() + let room = try participant.requireRoom() + try await room.signalClient.sendUpdateLocalAudioTrack(trackSid: self.sid, + features: newFeatures) + } } } From 9c69fc30aee6a7b38e2596e066326f361b93d937 Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:44:58 +0800 Subject: [PATCH 5/7] Organize into file AudioCustomProcessingDelegate --- .../AudioCustomProcessingDelegate.swift | 81 +++++++++++++++++++ Sources/LiveKit/Track/AudioManager.swift | 58 ------------- 2 files changed, 81 insertions(+), 58 deletions(-) create mode 100644 Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift diff --git a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift new file mode 100644 index 000000000..1838febfb --- /dev/null +++ b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift @@ -0,0 +1,81 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +@objc +public protocol AudioCustomProcessingDelegate { + @objc optional + var audioProcessingName: String { get } + + @objc + func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) + + @objc + func audioProcessingProcess(audioBuffer: LKAudioBuffer) + + @objc + func audioProcessingRelease() +} + +class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate { + // + public var target: AudioCustomProcessingDelegate? { _state.target } + + private struct State { + weak var target: AudioCustomProcessingDelegate? + } + + private var _state: StateSync + + init(target: AudioCustomProcessingDelegate? = nil) { + _state = StateSync(State(target: target)) + } + + public func set(target: AudioCustomProcessingDelegate?) { + _state.mutate { $0.target = target } + } + + func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) { + target?.audioProcessingInitialize(sampleRate: sampleRateHz, channels: channels) + } + + func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) { + target?.audioProcessingProcess(audioBuffer: LKAudioBuffer(audioBuffer: audioBuffer)) + } + + func audioProcessingRelease() { + target?.audioProcessingRelease() + } + + // Proxy the equality operators + + override func isEqual(_ object: Any?) -> Bool { + guard let other = object as? AudioCustomProcessingDelegateAdapter else { return false } + return target === other.target + } + + override var hash: Int { + guard let target else { return 0 } + return ObjectIdentifier(target).hashValue + } +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index cb987b495..80aa9ab82 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -57,64 +57,6 @@ public class LKAudioBuffer: NSObject { } } -@objc -public protocol AudioCustomProcessingDelegate { - @objc optional - var audioProcessingName: String { get } - - @objc - func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) - - @objc - func audioProcessingProcess(audioBuffer: LKAudioBuffer) - - @objc - func audioProcessingRelease() -} - -class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate { - // - public var target: AudioCustomProcessingDelegate? { _state.target } - - private struct State { - weak var target: AudioCustomProcessingDelegate? - } - - private var _state: StateSync - - init(target: AudioCustomProcessingDelegate? = nil) { - _state = StateSync(State(target: target)) - } - - public func set(target: AudioCustomProcessingDelegate?) { - _state.mutate { $0.target = target } - } - - func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) { - target?.audioProcessingInitialize(sampleRate: sampleRateHz, channels: channels) - } - - func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) { - target?.audioProcessingProcess(audioBuffer: LKAudioBuffer(audioBuffer: audioBuffer)) - } - - func audioProcessingRelease() { - target?.audioProcessingRelease() - } - - // Proxy the equality operators - - override func isEqual(_ object: Any?) -> Bool { - guard let other = object as? AudioCustomProcessingDelegateAdapter else { return false } - return target === other.target - } - - override var hash: Int { - guard let target else { return 0 } - return ObjectIdentifier(target).hashValue - } -} - // Audio Session Configuration related public class AudioManager: Loggable { // MARK: - Public From 6b5802298e5811bb27aab29bfe8b941b045d3d4d Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:49:35 +0800 Subject: [PATCH 6/7] Use constant for krisp processor name --- Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift | 2 ++ Sources/LiveKit/TrackPublications/LocalTrackPublication.swift | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift index 1838febfb..bb453e293 100644 --- a/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift +++ b/Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift @@ -22,6 +22,8 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif +public let kLiveKitKrispAudioProcessorName = "livekit_krisp_noise_cancellation" + @objc public protocol AudioCustomProcessingDelegate { @objc optional diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index a958af417..842781ca2 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -116,7 +116,7 @@ extension LocalTrackPublication { // Check if Krisp is enabled. if let processingDelegate = AudioManager.shared.capturePostProcessingDelegate, - processingDelegate.audioProcessingName == "krisp_noise_cancellation" + processingDelegate.audioProcessingName == kLiveKitKrispAudioProcessorName { newFeatures.insert(.tfEnhancedNoiseCancellation) } From d52851e0171c0b855dc775c2468d4a10b2278f97 Mon Sep 17 00:00:00 2001 From: hiroshihorie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:54:43 +0800 Subject: [PATCH 7/7] Add features from publish options also --- .../LiveKit/TrackPublications/LocalTrackPublication.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index 842781ca2..8270ea4a3 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -114,6 +114,11 @@ extension LocalTrackPublication { var newFeatures = audioTrack.captureOptions.toFeatures() + if let audioPublishOptions = audioTrack.publishOptions as? AudioPublishOptions { + // Combine features from publish options. + newFeatures.formUnion(audioPublishOptions.toFeatures()) + } + // Check if Krisp is enabled. if let processingDelegate = AudioManager.shared.capturePostProcessingDelegate, processingDelegate.audioProcessingName == kLiveKitKrispAudioProcessorName