Enhance API for configuring simulcast. Fix switching between available simulcast encodings

assets/js/const.ts

@@ -1,26 +1,26 @@
 // const TEMPORAL_LAYERS_COUNT = 2;
 
-export const simulcastConfig: RTCRtpTransceiverInit = {
+export const simulcastTransceiverConfig: RTCRtpTransceiverInit = {
   direction: "sendonly",
   // keep this array from low resolution to high resolution
   // in other case lower resolution encoding can get
   // higher max_bitrate
   sendEncodings: [
     {
       rid: "l",
-      active: true,
+      active: false,
       // maxBitrate: 4_000_000,
       scaleResolutionDownBy: 4.0,
       //   scalabilityMode: "L1T" + TEMPORAL_LAYERS_COUNT,
     },
     {
       rid: "m",
-      active: true,
+      active: false,
       scaleResolutionDownBy: 2.0,
     },
     {
       rid: "h",
-      active: true,
+      active: false,
       // maxBitrate: 4_000_000,
       // scalabilityMode: "L1T" + TEMPORAL_LAYERS_COUNT,
     },

assets/js/membraneWebRTC.ts

@@ -7,7 +7,7 @@ import {
   serializeMediaEvent,
 } from "./mediaEvent";
 import { v4 as uuidv4 } from "uuid";
-import { simulcastConfig } from "./const";
+import { simulcastTransceiverConfig } from "./const";
 
 /**
  * Interface describing Peer.
@@ -41,6 +41,22 @@ export interface MembraneWebRTCConfig {
   rtcConfig?: RTCConfiguration;
 }
 
+/**
+ * Simulcast configuration passed to {@link addTrack}.
+ */
+export interface SimulcastConfig {
+  /**
+   * Whether to simulcast track or not.
+   */
+  enabled: boolean;
+  /**
+   * List of initially active encodings.
+   * Encoding that is not present in this list might still be
+   * enabled using {@link enableTrackEncoding}.
+   */
+  active_encodings: TrackEncoding[];
+}
+
 /**
  * Track's context i.e. all data that can be useful when operating on track.
  */
@@ -60,9 +76,9 @@ export interface TrackContext {
    */
   trackId: string;
   /**
-   * Flag indicating whether track is a simulcast one or not.
+   * Simulcast configuration.
    */
-  isSimulcast: boolean;
+  simulcastConfig: SimulcastConfig;
   /**
    * Any info that was passed in {@link addTrack}.
    */
@@ -73,10 +89,16 @@ export interface TrackContext {
 
 /**
  * Type describing possible track encodings.
- * At the moment, if track was added as a simulcast one ({@link addTrack})
- * it will be transmitted to the server in two versions - low and high.
+ * `"h"` - original encoding
+ * `"m"` - original encoding scaled down by 2
+ * `"l"` - original encoding scaled down by 4
+ *
+ * Notice that to make all encodings work, the initial
+ * resolution has to be at least 1280x720.
+ * In other case, browser might not be able to scale
+ * some encodings down.
  */
-export type TrackEncoding = "l" | "h";
+export type TrackEncoding = "l" | "m" | "h";
 
 /**
  * Callbacks that has to be implemented by user.
@@ -292,7 +314,7 @@ export class MembraneWebRTC {
               stream: null,
               track: null,
               trackId,
-              isSimulcast: false,
+              simulcastConfig: { enabled: false, active_encodings: [] },
               metadata,
               peer,
               maxBandwidth: 0,
@@ -448,7 +470,7 @@ export class MembraneWebRTC {
     track: MediaStreamTrack,
     stream: MediaStream,
     trackMetadata: any = new Map(),
-    isSimulcast: boolean = false,
+    simulcastConfig: SimulcastConfig = { enabled: false, active_encodings: [] },
     maxBandwidth: BandwidthLimit = 0 // unlimited bandwidth
   ): string {
     if (this.getPeerId() === "") throw "Cannot add tracks before being accepted by the server";
@@ -462,7 +484,7 @@ export class MembraneWebRTC {
       trackId,
       peer: this.localPeer,
       metadata: trackMetadata,
-      isSimulcast,
+      simulcastConfig,
       maxBandwidth,
     };
     this.localTrackIdToTrack.set(trackId, trackContext);
@@ -485,12 +507,41 @@ export class MembraneWebRTC {
   }
 
   private addTrackToConnection = (trackContext: TrackContext) => {
+    let transceiverConfig = this.createTransceiverConfig(trackContext);
     const track = trackContext.track!!;
+    this.connection!.addTransceiver(track, transceiverConfig);
+  };
+
+  private createTransceiverConfig(trackContext: TrackContext): RTCRtpTransceiverInit {
     let transceiverConfig: RTCRtpTransceiverInit;
 
-    if (trackContext.isSimulcast) {
-      transceiverConfig = track.kind === "audio" ? { direction: "sendonly" } : simulcastConfig;
-      this.disabledTrackEncodings.set(trackContext.trackId, []);
+    if (trackContext.track!!.kind === "audio") {
+      transceiverConfig = this.createAudioTransceiverConfig(trackContext);
+    } else {
+      transceiverConfig = this.createVideoTransceiverConfig(trackContext);
+    }
+
+    return transceiverConfig;
+  }
+
+  private createAudioTransceiverConfig(_trackContext: TrackContext): RTCRtpTransceiverInit {
+    return { direction: "sendonly" };
+  }
+
+  private createVideoTransceiverConfig(trackContext: TrackContext): RTCRtpTransceiverInit {
+    let transceiverConfig: RTCRtpTransceiverInit;
+    if (trackContext.simulcastConfig.enabled) {
+      transceiverConfig = simulcastTransceiverConfig;
+      let trackActiveEncodings = trackContext.simulcastConfig.active_encodings;
+      let disabledTrackEncodings: TrackEncoding[] = [];
+      transceiverConfig.sendEncodings?.forEach((encoding) => {
+        if (trackActiveEncodings.includes(encoding.rid! as TrackEncoding)) {
+          encoding.active = true;
+        } else {
+          disabledTrackEncodings.push(encoding.rid! as TrackEncoding);
+        }
+      });
+      this.disabledTrackEncodings.set(trackContext.trackId, disabledTrackEncodings);
     } else {
       transceiverConfig = {
         direction: "sendonly",
@@ -506,8 +557,8 @@ export class MembraneWebRTC {
       }
     }
 
-    this.connection!.addTransceiver(track, transceiverConfig);
-  };
+    return transceiverConfig;
+  }
 
   /**
    * Replaces a track that is being sent to the RTC Engine.
@@ -993,7 +1044,7 @@ export class MembraneWebRTC {
         peer: peer,
         trackId,
         metadata,
-        isSimulcast: false,
+        simulcastConfig: { enabled: false, active_encodings: [] },
       };
 
       this.trackIdToTrack.set(trackId, trackContext);

guides/simulcast.md

@@ -4,29 +4,28 @@ Simulcast is a technique where a client sends multiple encodings of the same vid
 
 * receiver awailable bandwidth
 * receiver preferences (e.g. explicit request to receive video in HD resolution instead of FHD)
-* UI layaout (e.g. videos being displayed in smaller video tiles will be sent in lower resolution)
+* UI layaout (e.g. videos being displayed in smaller video tiles will be sent in a lower resolution)
 
-At the moment, Membrane supports only receiver preferences i.e. receiver can chose which encoding it is willing to receive. Additionaly, sender can turn off/on specific encoding. Membrane RTC Engine will detect changes and switch to another available encoding.Simulcast is a technique where client sends multiple encodings (different resolutions) of the same
-video to a server and the server forwards proper encoding to each other client basing on 
-client preferences, network bandwidth or UI layaout.
+At the moment, Membrane supports only receiver preferences i.e. receiver can chose which encoding it is willing to receive. Additionaly, sender can turn off/on specific encoding. Membrane RTC Engine will detect changes and switch to another available encoding.
 
 ## Turning simulcast on/off
 
-On the client side simulcast can be enabled while adding new track e.g.:
+On the client side simulcast can be enabled while adding a new track e.g.:
 
 ```ts
     // create MembraneWebRTC class instance
     // ...
     // add simulcasted track
-    let trackId = webrtc.addTrack(track, stream, {}, true);
+    let trackId = webrtc.addTrack(track, stream, {}, {enabled: true, active_encodings: ["l", "m", "h"]});
 ```
 
 This will add a new track that will be sent in three versions:
 * original (identified as `h`)
 * original scaled down by 2 (identified as `m`)
 * original scaled down by 4 (identified as `l`)
 
-Those settings are not configurable at the moment.
+You can turn off some of the encodings by excluding them from `active_encodings` list.
+Encodings that are turned off might still be enabled using `enableTrackEncoding` function.
 
 > #### Minimal required resolution {: .warning}
 >
@@ -36,9 +35,24 @@ Those settings are not configurable at the moment.
 > passed to [`getUserMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) 
 > or [`getDisplayMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia).
 
-On the server side, simulcast can be disabled while adding new WebRTC Endpoint by setting its `simulcast?` 
-option to `false`.
-This will result in rejecting all incoming simulcast tracks i.e. client will not send them.
+On the server side, simulcast can be configured while adding new WebRTC Endpoint by setting its `simulcast_config` option.
+
+For example
+
+```elixir
+%WebRTC{
+  rtc_engine: rtc_engine,
+  # ...
+  simulcast_config: %SimulcastConfig{
+    enabled: true,
+    default_encoding: fn %Track{simulcast_encodings: _simulcast_encodings} -> "m" end
+  }
+}
+```
+
+Here we turn simulcast on and choose medium encoding for each track to be forwarded to the client.
+
+On the other hand, setting `enabled` to `false` will result in rejecting all incoming simulcast tracks i.e. client will not send them to the server.
 
 ## Disabling and enabling specific track encoding
 
@@ -58,7 +72,7 @@ Disabled encoding can be turned on again using `enableTrackEncoding` function.
 Membrane RTC Engine tracks encoding activity. 
 Therefore, when some encoding is turned off, RTC Engine will detect this and switch to 
 the highest awailable encoding.
-If turned off encoding returns, RTC Engine will switch back to it.
+When disabled encoding becomes active again, RTC Engine will switch back to it.
 
 ## Selecting encoding to receive
 

lib/membrane_rtc_engine/endpoints/hls_endpoint.ex

@@ -80,25 +80,18 @@ if Enum.all?(
 
     @impl true
     def handle_other({:new_tracks, tracks}, ctx, state) do
-      new_tracks = Map.new(tracks, &{&1.id, &1})
-
-      subscriptions =
-        tracks
-        |> Enum.filter(fn track -> :raw in track.format end)
-        |> Enum.map(fn track -> {track.id, :raw} end)
-
       {:endpoint, endpoint_id} = ctx.name
+      tracks = Enum.filter(tracks, fn track -> :raw in track.format end)
 
-      case Engine.subscribe(state.rtc_engine, subscriptions, endpoint_id) do
-        :ok ->
-          {:ok, Map.update!(state, :tracks, &Map.merge(&1, new_tracks))}
+      Enum.each(tracks, fn track ->
+        case Engine.subscribe(state.rtc_engine, endpoint_id, track.id, :raw) do
+          :ok ->
+            {:ok, put_in(state, [:tracks, track.id], track)}
 
-        {:error, track_id, reason} ->
-          raise "Subscription fails on track: #{track_id} because of #{reason}"
-
-        {:error, :timeout} ->
-          raise "Timeout subscribing on track in Engine with pid #{state.rtc_engine}"
-      end
+          {:error, reason} ->
+            raise "Couldn't subscribe for track: #{inspect(track.id)}. Reason: #{inspect(reason)}"
+        end
+      end)
     end
 
     @impl true