From 7d7975b8bbdee1cd184a8c71e6a04d7a6bd7073e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 7 Oct 2021 13:47:50 -0700 Subject: [PATCH 1/6] Fix connecting to a call without a webcam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 12 ++++++++++-- src/webrtc/mediaHandler.ts | 25 ++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e13e2644f74..9982a0c0780 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -562,8 +562,8 @@ export class MatrixCall extends EventEmitter { this.feeds.push(new CallFeed({ client: this.client, roomId: this.roomId, - audioMuted: false, - videoMuted: false, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, userId, stream, purpose, @@ -993,6 +993,10 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + if (!this.hasLocalUserMediaVideoTrack && !muted) { await this.upgradeCall(false, true); return this.isLocalVideoMuted(); @@ -1021,6 +1025,10 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + if (!this.hasLocalUserMediaAudioTrack && !muted) { await this.upgradeCall(true, false); return this.isMicrophoneMuted(); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 2792f1e180f..a900ffe16c4 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -43,23 +43,42 @@ export class MediaHandler { this.videoInput = deviceId; } + public async hasAudioDevice() { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter(device => device.kind === "audioinput").length > 0; + } + + public async hasVideoDevice() { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter(device => device.kind === "videoinput").length > 0; + } + /** * @returns {MediaStream} based on passed parameters */ public async getUserMediaStream(audio: boolean, video: boolean): Promise { + const devices = await navigator.mediaDevices.enumerateDevices(); + + const audioDevices = devices.filter(device => device.kind === "audioinput"); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + const shouldRequestAudio = audio && audioDevices.length > 0; + const shouldRequestVideo = video && videoDevices.length > 0; + let stream: MediaStream; // Find a stream with matching tracks const matchingStream = this.userMediaStreams.find((stream) => { - if (audio !== (stream.getAudioTracks().length > 0)) return false; - if (video !== (stream.getVideoTracks().length > 0)) return false; + if (shouldRequestAudio !== (stream.getAudioTracks().length > 0)) return false; + if (shouldRequestVideo !== (stream.getVideoTracks().length > 0)) return false; + return true; }); if (matchingStream) { logger.log("Cloning user media stream", matchingStream.id); stream = matchingStream.clone(); } else { - const constraints = this.getUserMediaContraints(audio, video); + const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); } From cb8a260b9c60a0756bb7a7b6b3bdde9f93462e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 9 Oct 2021 08:36:19 +0200 Subject: [PATCH 2/6] Improve handling when we don't have a camera MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/mediaHandler.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index a900ffe16c4..4a20823bd48 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -43,12 +43,12 @@ export class MediaHandler { this.videoInput = deviceId; } - public async hasAudioDevice() { + public async hasAudioDevice(): Promise { const devices = await navigator.mediaDevices.enumerateDevices(); return devices.filter(device => device.kind === "audioinput").length > 0; } - public async hasVideoDevice() { + public async hasVideoDevice(): Promise { const devices = await navigator.mediaDevices.enumerateDevices(); return devices.filter(device => device.kind === "videoinput").length > 0; } @@ -57,13 +57,8 @@ export class MediaHandler { * @returns {MediaStream} based on passed parameters */ public async getUserMediaStream(audio: boolean, video: boolean): Promise { - const devices = await navigator.mediaDevices.enumerateDevices(); - - const audioDevices = devices.filter(device => device.kind === "audioinput"); - const videoDevices = devices.filter(device => device.kind === "videoinput"); - - const shouldRequestAudio = audio && audioDevices.length > 0; - const shouldRequestVideo = video && videoDevices.length > 0; + const shouldRequestAudio = audio && await this.hasAudioDevice(); + const shouldRequestVideo = video && await this.hasVideoDevice(); let stream: MediaStream; From 4b679c50565f8f975a4f130326f0de6ceb514235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 9 Oct 2021 08:36:49 +0200 Subject: [PATCH 3/6] Try to answer without video if we can't access it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9982a0c0780..bc3e4ff286c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -752,19 +752,29 @@ export class MatrixCall extends EventEmitter { logger.debug(`Answering call ${this.callId}`); if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + const prevState = this.state; + const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); + const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); + this.setState(CallState.WaitLocalMedia); this.waitForLocalAVStream = true; try { const mediaStream = await this.client.getMediaHandler().getUserMediaStream( - this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"), - this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"), + answerWithAudio, answerWithVideo, ); this.waitForLocalAVStream = false; this.gotUserMediaForAnswer(mediaStream); } catch (e) { - this.getUserMediaFailed(e); - return; + if (answerWithVideo) { + // Try to answer without video + this.setState(prevState); + this.waitForLocalAVStream = false; + await this.answer(answerWithAudio, false); + } else { + this.getUserMediaFailed(e); + return; + } } } else if (this.waitForLocalAVStream) { this.setState(CallState.WaitLocalMedia); From 74a875aa234bdb5e7976ab1f61589ddd1591926e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 9 Oct 2021 08:57:38 +0200 Subject: [PATCH 4/6] Add MockMediaDeviceInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 00ca14892ee..61b58ac305c 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -94,6 +94,12 @@ class MockMediaStream { addEventListener() {} } +class MockMediaDeviceInfo { + constructor( + public kind: "audio" | "video", + ) {} +} + describe('Call', function() { let client; let call; @@ -110,6 +116,8 @@ describe('Call', function() { mediaDevices: { // @ts-ignore Mock getUserMedia: () => new MockMediaStream("local_stream"), + // @ts-ignore Mock + enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], }, }; From 68dbe959bb9ac06598d76d0c9352e71bec97259c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 9 Oct 2021 08:57:46 +0200 Subject: [PATCH 5/6] Add a logline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bc3e4ff286c..7afa8f08ae6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -768,6 +768,7 @@ export class MatrixCall extends EventEmitter { } catch (e) { if (answerWithVideo) { // Try to answer without video + logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); this.setState(prevState); this.waitForLocalAVStream = false; await this.answer(answerWithAudio, false); From 3aefc9f02e68fd0ac47d2e91c64ce246e6dd012c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 14 Oct 2021 08:46:36 +0200 Subject: [PATCH 6/6] Add tests for falling back to answering with no video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 61b58ac305c..9e12db20f47 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -100,6 +100,11 @@ class MockMediaDeviceInfo { ) {} } +class MockMediaHandler { + getUserMediaStream() { return new MockMediaStream("mock_stream_from_media_handler"); } + stopUserMediaStream() {} +} + describe('Call', function() { let client; let call; @@ -137,6 +142,8 @@ describe('Call', function() { // We just stub out sendEvent: we're not interested in testing the client's // event sending code here client.client.sendEvent = () => {}; + client.client.mediaHandler = new MockMediaHandler; + client.client.getMediaHandler = () => client.client.mediaHandler; client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); call = new MatrixCall({ client: client.client, @@ -376,4 +383,16 @@ describe('Call', function() { call.setScreensharingEnabled(true); expect(call.setScreensharingEnabledWithoutMetadataSupport).toHaveBeenCalled(); }); + + it("should fallback to answering with no video", async () => { + await client.httpBackend.flush(); + + call.shouldAnswerWithMediaType = (wantedValue: boolean) => wantedValue; + client.client.mediaHandler.getUserMediaStream = jest.fn().mockRejectedValue("reject"); + + await call.answer(true, true); + + expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(1, true, true); + expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(2, true, false); + }); });