Send audio track features CLT-208

Sources/LiveKit/Core/SignalClient.swift

@@ -510,6 +510,17 @@ extension SignalClient {
         try await _sendRequest(r)
     }
 
+    func sendUpdateLocalAudioTrack(trackSid: Track.Sid, features: Set<Livekit_AudioTrackFeature>) async throws {
+        let r = Livekit_SignalRequest.with {
+            $0.updateAudioTrack = Livekit_UpdateLocalAudioTrack.with {
+                $0.trackSid = trackSid.stringValue
+                $0.features = Array(features)
+            }
+        }
+
+        try await _sendRequest(r)
+    }
+
     func sendSyncState(answer: Livekit_SessionDescription?,
                        offer: Livekit_SessionDescription?,
                        subscription: Livekit_UpdateSubscription,

Sources/LiveKit/Protocols/AudioCustomProcessingDelegate.swift

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2024 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+#if swift(>=5.9)
+internal import LiveKitWebRTC
+#else
+@_implementationOnly import LiveKitWebRTC
+#endif
+
+public let kLiveKitKrispAudioProcessorName = "livekit_krisp_noise_cancellation"
+
+@objc
+public protocol AudioCustomProcessingDelegate {
+    @objc optional
+    var audioProcessingName: String { get }
+
+    @objc
+    func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int)
+
+    @objc
+    func audioProcessingProcess(audioBuffer: LKAudioBuffer)
+
+    @objc
+    func audioProcessingRelease()
+}
+
+class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate {
+    //
+    public var target: AudioCustomProcessingDelegate? { _state.target }
+
+    private struct State {
+        weak var target: AudioCustomProcessingDelegate?
+    }
+
+    private var _state: StateSync<State>
+
+    init(target: AudioCustomProcessingDelegate? = nil) {
+        _state = StateSync(State(target: target))
+    }
+
+    public func set(target: AudioCustomProcessingDelegate?) {
+        _state.mutate { $0.target = target }
+    }
+
+    func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) {
+        target?.audioProcessingInitialize(sampleRate: sampleRateHz, channels: channels)
+    }
+
+    func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) {
+        target?.audioProcessingProcess(audioBuffer: LKAudioBuffer(audioBuffer: audioBuffer))
+    }
+
+    func audioProcessingRelease() {
+        target?.audioProcessingRelease()
+    }
+
+    // Proxy the equality operators
+
+    override func isEqual(_ object: Any?) -> Bool {
+        guard let other = object as? AudioCustomProcessingDelegateAdapter else { return false }
+        return target === other.target
+    }
+
+    override var hash: Int {
+        guard let target else { return 0 }
+        return ObjectIdentifier(target).hashValue
+    }
+}

Sources/LiveKit/Track/AudioManager.swift

@@ -16,6 +16,7 @@
 
 import Accelerate
 import AVFoundation
+import Combine
 
 #if swift(>=5.9)
 internal import LiveKitWebRTC
@@ -56,45 +57,6 @@ public class LKAudioBuffer: NSObject {
     }
 }
 
-@objc
-public protocol AudioCustomProcessingDelegate {
-    func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int)
-    func audioProcessingProcess(audioBuffer: LKAudioBuffer)
-    func audioProcessingRelease()
-}
-
-class AudioCustomProcessingDelegateAdapter: NSObject, LKRTCAudioCustomProcessingDelegate {
-    weak var target: AudioCustomProcessingDelegate?
-
-    init(target: AudioCustomProcessingDelegate? = nil) {
-        self.target = target
-    }
-
-    func audioProcessingInitialize(sampleRate sampleRateHz: Int, channels: Int) {
-        target?.audioProcessingInitialize(sampleRate: sampleRateHz, channels: channels)
-    }
-
-    func audioProcessingProcess(audioBuffer: LKRTCAudioBuffer) {
-        target?.audioProcessingProcess(audioBuffer: LKAudioBuffer(audioBuffer: audioBuffer))
-    }
-
-    func audioProcessingRelease() {
-        target?.audioProcessingRelease()
-    }
-
-    // Proxy the equality operators
-
-    override func isEqual(_ object: Any?) -> Bool {
-        guard let other = object as? AudioCustomProcessingDelegateAdapter else { return false }
-        return target === other.target
-    }
-
-    override var hash: Int {
-        guard let target else { return 0 }
-        return ObjectIdentifier(target).hashValue
-    }
-}
-
 // Audio Session Configuration related
 public class AudioManager: Loggable {
     // MARK: - Public
@@ -173,14 +135,19 @@ public class AudioManager: Loggable {
         return adapter
     }()
 
+    let capturePostProcessingDelegateSubject = CurrentValueSubject<AudioCustomProcessingDelegate?, Never>(nil)
+
     public var capturePostProcessingDelegate: AudioCustomProcessingDelegate? {
         get { capturePostProcessingDelegateAdapter.target }
-        set { capturePostProcessingDelegateAdapter.target = newValue }
+        set {
+            capturePostProcessingDelegateAdapter.set(target: newValue)
+            capturePostProcessingDelegateSubject.send(newValue)
+        }
     }
 
     public var renderPreProcessingDelegate: AudioCustomProcessingDelegate? {
         get { renderPreProcessingDelegateAdapter.target }
-        set { renderPreProcessingDelegateAdapter.target = newValue }
+        set { renderPreProcessingDelegateAdapter.set(target: newValue) }
     }
 
     // MARK: - AudioDeviceModule

Sources/LiveKit/Track/Local/LocalAudioTrack.swift

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import Combine
 import Foundation
 
 #if swift(>=5.9)
@@ -24,11 +25,17 @@ internal import LiveKitWebRTC
 
 @objc
 public class LocalAudioTrack: Track, LocalTrack, AudioTrack {
+    /// ``AudioCaptureOptions`` used to create this track.
+    let captureOptions: AudioCaptureOptions
+
     init(name: String,
          source: Track.Source,
          track: LKRTCMediaStreamTrack,
-         reportStatistics: Bool)
+         reportStatistics: Bool,
+         captureOptions: AudioCaptureOptions)
     {
+        self.captureOptions = captureOptions
+
         super.init(name: name,
                    kind: .audio,
                    source: source,
@@ -62,7 +69,8 @@ public class LocalAudioTrack: Track, LocalTrack, AudioTrack {
         return LocalAudioTrack(name: name,
                                source: .microphone,
                                track: rtcTrack,
-                               reportStatistics: reportStatistics)
+                               reportStatistics: reportStatistics,
+                               captureOptions: options)
     }
 
     @discardableResult

Sources/LiveKit/TrackPublications/LocalTrackPublication.swift

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import Combine
 import Foundation
 
 @objc
@@ -26,6 +27,19 @@ public class LocalTrackPublication: TrackPublication {
 
     // MARK: - Private
 
+    private var _cancellable: AnyCancellable?
+
+    override init(info: Livekit_TrackInfo, participant: Participant) {
+        super.init(info: info, participant: participant)
+
+        // Watch audio manager processor changes.
+        _cancellable = AudioManager.shared.capturePostProcessingDelegateSubject.sink { [weak self] _ in
+            self?.sendAudioTrackFeatures()
+        }
+    }
+
+    // MARK: - Private
+
     private let _debounce = Debounce(delay: 0.1)
 
     public func mute() async throws {
@@ -57,6 +71,8 @@ public class LocalTrackPublication: TrackPublication {
             newLocalVideoTrack.capturer.add(delegate: self)
         }
 
+        sendAudioTrackFeatures()
+
         return oldValue
     }
 
@@ -92,6 +108,42 @@ extension LocalTrackPublication: VideoCapturerDelegate {
 }
 
 extension LocalTrackPublication {
+    func sendAudioTrackFeatures() {
+        // Only proceed if audio track.
+        guard let audioTrack = track as? LocalAudioTrack else { return }
+
+        var newFeatures = audioTrack.captureOptions.toFeatures()
+
+        if let audioPublishOptions = audioTrack.publishOptions as? AudioPublishOptions {
+            // Combine features from publish options.
+            newFeatures.formUnion(audioPublishOptions.toFeatures())
+        }
+
+        // Check if Krisp is enabled.
+        if let processingDelegate = AudioManager.shared.capturePostProcessingDelegate,
+           processingDelegate.audioProcessingName == kLiveKitKrispAudioProcessorName
+        {
+            newFeatures.insert(.tfEnhancedNoiseCancellation)
+        }
+
+        let didUpdateFeatures = _state.mutate {
+            let oldFeatures = $0.audioTrackFeatures
+            $0.audioTrackFeatures = newFeatures
+            return oldFeatures != newFeatures
+        }
+
+        if didUpdateFeatures {
+            log("Sending audio track features: \(newFeatures)")
+            // Send if features updated.
+            Task.detached { [newFeatures] in
+                let participant = try await self.requireParticipant()
+                let room = try participant.requireRoom()
+                try await room.signalClient.sendUpdateLocalAudioTrack(trackSid: self.sid,
+                                                                      features: newFeatures)
+            }
+        }
+    }
+
     func recomputeSenderParameters() {
         guard let track = track as? LocalVideoTrack,
               let sender = track._state.rtpSender else { return }

Sources/LiveKit/TrackPublications/TrackPublication.swift

@@ -90,6 +90,8 @@ public class TrackPublication: NSObject, ObservableObject, Loggable {
         var encryptionType: EncryptionType = .none
 
         var latestInfo: Livekit_TrackInfo?
+
+        var audioTrackFeatures: Set<Livekit_AudioTrackFeature>?
     }
 
     let _state: StateSync<State>

Sources/LiveKit/Types/Options/AudioCaptureOptions.swift

@@ -83,3 +83,14 @@ public class AudioCaptureOptions: NSObject, CaptureOptions {
         return hasher.finalize()
     }
 }
+
+// Internal
+extension AudioCaptureOptions {
+    func toFeatures() -> Set<Livekit_AudioTrackFeature> {
+        Set([
+            echoCancellation ? .tfEchoCancellation : nil,
+            noiseSuppression ? .tfNoiseSuppression : nil,
+            autoGainControl ? .tfAutoGainControl : nil,
+        ].compactMap { $0 })
+    }
+}

Sources/LiveKit/Types/Options/AudioPublishOptions.swift

@@ -61,3 +61,12 @@ public class AudioPublishOptions: NSObject, TrackPublishOptions {
         return hasher.finalize()
     }
 }
+
+// Internal
+extension AudioPublishOptions {
+    func toFeatures() -> Set<Livekit_AudioTrackFeature> {
+        Set([
+            !dtx ? .tfNoDtx : nil,
+        ].compactMap { $0 })
+    }
+}