From 49cf63213dee35dc025481444b315f056de6b816 Mon Sep 17 00:00:00 2001 From: Ashot Hovhannisyan Date: Wed, 13 Nov 2024 11:25:24 +0400 Subject: [PATCH] feat(ns) update Krisp to latest version * feat(noise-suppression): Add support for BVC model * feat(noise-suppression): Add support for extended BVC devices list This commit adds support for extending the list of allowed BVC devices in the noise suppression feature. It introduces a new file, `bvc-allowed-ext.txt`, which can be used to include additional BVC supported devices. * fix(noise-suppression): reuse the existing audioStream for Krisp initialization * update krisp configuration * refactor(krisp): Update Krisp JS SDK version and noise suppression configuration * refactor(config): update config default values, add the enabled flad in INoiseSuppressionConfig --- config.js | 37 ++++++++++- react/features/base/config/configType.ts | 20 +++++- .../NoiseSuppressionEffect.ts | 66 ++++++++++++++----- 3 files changed, 101 insertions(+), 22 deletions(-) diff --git a/config.js b/config.js index 12ec656a73cc..ba1cac71a04d 100644 --- a/config.js +++ b/config.js @@ -209,14 +209,45 @@ var config = { // installation. Specifically, these files are needed: // - https://meet.example.com/libs/krisp/krisp.mjs // - https://meet.example.com/libs/krisp/models/model_8.kw - // - https://meet.example.com/libs/krisp/models/model_16.kw - // - https://meet.example.com/libs/krisp/models/model_32.kw - // NOTE: Krisp JS SDK v1.0.9 was tested. + // - https://meet.example.com/libs/krisp/models/model_nc.kw + // - https://meet.example.com/libs/krisp/models/model_bvc.kw + // - https://meet.example.com/libs/krisp/assets/bvc-allowed.txt + // In case when you have known BVC supported devices and you want to extend allowed devices list + // - https://meet.example.com/libs/krisp/assets/bvc-allowed-ext.txt + // In case when you have known BVC supported devices and you want to extend allowed devices list + // - https://meet.example.com/libs/krisp/models/model_inbound_8.kw + // - https://meet.example.com/libs/krisp/models/model_inbound_16.kw + // In case when you want to use inbound noise suppression models + // NOTE: Krisp JS SDK v2.0.0 was tested. // noiseSuppression: { // krisp: { // enabled: false, // logProcessStats: false, // debugLogs: false, + // useBVC: false, + // bufferOverflowMS: 1000, + // inboundModels: { + // modelInbound8: 'model_inbound_8.kef', + // modelInbound16: 'model_inbound_16.kef', + // }, + // preloadInboundModels: { + // modelInbound8: 'model_inbound_8.kef', + // modelInbound16: 'model_inbound_16.kef', + // }, + // preloadModels: { + // modelBVC: 'model_bvc.kef', + // model8: 'model_8.kef', + // modelNC: 'model_nc_mq.kef', + // }, + // models: { + // modelBVC: 'model_bvc.kef', + // model8: 'model_8.kef', + // modelNV: 'model_nc_mq.kef', + // }, + // bvc: { + // allowedDevices: 'bvc-allowed.txt', + // allowedDevicesExt: 'bvc-allowed-ext.txt', + // } // }, // }, diff --git a/react/features/base/config/configType.ts b/react/features/base/config/configType.ts index 68e9ed8b0017..34e34c192800 100644 --- a/react/features/base/config/configType.ts +++ b/react/features/base/config/configType.ts @@ -124,11 +124,27 @@ export interface IDeeplinkingConfig { ios?: IDeeplinkingMobileConfig; } +export type PartialRecord = { + [P in K]?: T; +}; + export interface INoiseSuppressionConfig { krisp?: { - debugLogs?: boolean; - enabled?: boolean; + bufferOverflowMS?: number; + bvc?: { + allowedDevices?: string; + allowedDevicesExt?: string; + }; + debugLogs: boolean; + enableSessionStats?: boolean; + enabled: boolean; + inboundModels?: PartialRecord; logProcessStats?: boolean; + models?: PartialRecord; + preloadInboundModels?: PartialRecord; + preloadModels?: PartialRecord; + useBVC?: boolean; + useSharedArrayBuffer?: boolean; }; } diff --git a/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts b/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts index 9b99648266d1..9c63914c4890 100644 --- a/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts +++ b/react/features/stream-effects/noise-suppression/NoiseSuppressionEffect.ts @@ -90,7 +90,7 @@ export class NoiseSuppressionEffect { let init; if (this._options?.krisp?.enabled) { - init = _initializeKrisp(this._options).then(filterNode => { + init = _initializeKrisp(this._options, audioStream).then(filterNode => { this._noiseSuppressorNode = filterNode; if (krispState.filterNodeReady) { @@ -167,26 +167,52 @@ export class NoiseSuppressionEffect { * Initializes the Krisp SDK and creates the filter node. * * @param {INoiseSuppressionConfig} options - Krisp options. + * @param {MediaStream} stream - Audio stream which will be mixed with _mixAudio. * * @returns {Promise} */ -async function _initializeKrisp(options: INoiseSuppressionConfig): Promise { +async function _initializeKrisp( + options: INoiseSuppressionConfig, + stream: MediaStream +): Promise { await audioContext.resume(); if (!krispState.sdk) { const baseUrl = `${getBaseUrl()}libs/krisp`; const { default: KrispSDK } = await import(/* webpackIgnore: true */ `${baseUrl}/krispsdk.mjs`); - krispState.sdk = new KrispSDK({ - params: { + const ncParams = { + krisp: { models: { - model8: `${baseUrl}/models/model_8.kw`, - model16: `${baseUrl}/models/model_16.kw`, - model32: `${baseUrl}/models/model_32.kw` + modelBVC: `${baseUrl}/models/${options?.krisp?.models?.modelBVC}`, + model8: `${baseUrl}/models/${options?.krisp?.models?.model8}`, + modelNC: `${baseUrl}/models/${options?.krisp?.models?.modelNC}` + }, + logProcessStats: !options?.krisp?.logProcessStats, + debugLogs: !options?.krisp?.debugLogs, + useBVC: !options?.krisp?.useBVC, + bvc: { + allowedDevices: `${baseUrl}/assets/${options?.krisp?.bvc?.allowedDevices}`, + allowedDevicesExt: `${baseUrl}/assets/${options?.krisp?.bvc?.allowedDevicesExt}` + }, + inboundModels: { + modelInbound8: `${baseUrl}/models/${options?.krisp?.inboundModels?.modelInbound8}`, + modelInbound16: `${baseUrl}/models/${options?.krisp?.inboundModels?.modelInbound16}` + }, + preloadModels: { + modelBVC: `${baseUrl}/models/${options?.krisp?.preloadModels?.modelBVC}`, + model8: `${baseUrl}/models/${options?.krisp?.preloadModels?.model8}`, + modelNC: `${baseUrl}/models/${options?.krisp?.preloadModels?.modelNC}` }, - logProcessStats: options?.krisp?.logProcessStats, - debugLogs: options?.krisp?.debugLogs - }, + preloadInboundModels: { + modelInbound8: `${baseUrl}/models/${options?.krisp?.preloadInboundModels?.modelInbound8}`, + modelInbound16: `${baseUrl}/models/${options?.krisp?.preloadInboundModels?.modelInbound16}` + } + } + }; + + krispState.sdk = new KrispSDK({ + params: ncParams.krisp, callbacks: {} }); } @@ -201,14 +227,20 @@ async function _initializeKrisp(options: INoiseSuppressionConfig): Promise