Enhance API for configuring simulcast. Fix switching between available simulcast encodings

assets/js/const.ts

@@ -1,26 +1,26 @@
 // const TEMPORAL_LAYERS_COUNT = 2;
 
-export const simulcastConfig: RTCRtpTransceiverInit = {
+export const simulcastTransceiverConfig: RTCRtpTransceiverInit = {
   direction: "sendonly",
   // keep this array from low resolution to high resolution
   // in other case lower resolution encoding can get
   // higher max_bitrate
   sendEncodings: [
     {
       rid: "l",
-      active: true,
+      active: false,
       // maxBitrate: 4_000_000,
       scaleResolutionDownBy: 4.0,
       //   scalabilityMode: "L1T" + TEMPORAL_LAYERS_COUNT,
     },
     {
       rid: "m",
-      active: true,
+      active: false,
       scaleResolutionDownBy: 2.0,
     },
     {
       rid: "h",
-      active: true,
+      active: false,
       // maxBitrate: 4_000_000,
       // scalabilityMode: "L1T" + TEMPORAL_LAYERS_COUNT,
     },

assets/js/membraneWebRTC.ts

@@ -7,7 +7,7 @@ import {
   serializeMediaEvent,
 } from "./mediaEvent";
 import { v4 as uuidv4 } from "uuid";
-import { simulcastConfig } from "./const";
+import { simulcastTransceiverConfig } from "./const";
 
 /**
  * Interface describing Peer.
@@ -41,6 +41,22 @@ export interface MembraneWebRTCConfig {
   rtcConfig?: RTCConfiguration;
 }
 
+/**
+ * Simulcast configuration passed to {@link addTrack}.
+ */
+export interface SimulcastConfig {
+  /**
+   * Whether to simulcast track or not.
+   */
+  enabled: boolean;
+  /**
+   * List of initially active encodings.
+   * Encoding that is not present in this list might still be
+   * enabled using {@link enableTrackEncoding}.
+   */
+  active_encodings: TrackEncoding[];
+}
+
 /**
  * Track's context i.e. all data that can be useful when operating on track.
  */
@@ -60,9 +76,9 @@ export interface TrackContext {
    */
   trackId: string;
   /**
-   * Flag indicating whether track is a simulcast one or not.
+   * Simulcast configuration.
    */
-  isSimulcast: boolean;
+  simulcastConfig: SimulcastConfig;
   /**
    * Any info that was passed in {@link addTrack}.
    */
@@ -73,10 +89,16 @@ export interface TrackContext {
 
 /**
  * Type describing possible track encodings.
- * At the moment, if track was added as a simulcast one ({@link addTrack})
- * it will be transmitted to the server in two versions - low and high.
+ * `"h"` - original encoding
+ * `"m"` - original encoding scaled down by 2
+ * `"l"` - original encoding scaled down by 4
+ *
+ * Notice that to make all encodings work, the initial
+ * resolution has to be at least 1280x720.
+ * In other case, browser might not be able to scale
+ * some encodings down.
  */
-export type TrackEncoding = "l" | "h";
+export type TrackEncoding = "l" | "m" | "h";
 
 /**
  * Callbacks that has to be implemented by user.
@@ -292,7 +314,7 @@ export class MembraneWebRTC {
               stream: null,
               track: null,
               trackId,
-              isSimulcast: false,
+              simulcastConfig: { enabled: false, active_encodings: [] },
               metadata,
               peer,
               maxBandwidth: 0,
@@ -448,7 +470,7 @@ export class MembraneWebRTC {
     track: MediaStreamTrack,
     stream: MediaStream,
     trackMetadata: any = new Map(),
-    isSimulcast: boolean = false,
+    simulcastConfig: SimulcastConfig = { enabled: false, active_encodings: [] },
     maxBandwidth: BandwidthLimit = 0 // unlimited bandwidth
   ): string {
     if (this.getPeerId() === "") throw "Cannot add tracks before being accepted by the server";
@@ -462,7 +484,7 @@ export class MembraneWebRTC {
       trackId,
       peer: this.localPeer,
       metadata: trackMetadata,
-      isSimulcast,
+      simulcastConfig,
       maxBandwidth,
     };
     this.localTrackIdToTrack.set(trackId, trackContext);
@@ -488,9 +510,20 @@ export class MembraneWebRTC {
     const track = trackContext.track!!;
     let transceiverConfig: RTCRtpTransceiverInit;
 
-    if (trackContext.isSimulcast) {
-      transceiverConfig = track.kind === "audio" ? { direction: "sendonly" } : simulcastConfig;
-      this.disabledTrackEncodings.set(trackContext.trackId, []);
+    if (trackContext.simulcastConfig.enabled) {
+      transceiverConfig =
+        track.kind === "audio" ? { direction: "sendonly" } : simulcastTransceiverConfig;
+      let disabledTrackEncodings: TrackEncoding[] = [];
+      transceiverConfig.sendEncodings?.forEach((encoding) => {
+        if (
+          trackContext.simulcastConfig.active_encodings.includes(encoding.rid! as TrackEncoding)
+        ) {
+          encoding.active = true;
+        } else {
+          disabledTrackEncodings.push(encoding.rid! as TrackEncoding);
+        }
+      });
+      this.disabledTrackEncodings.set(trackContext.trackId, disabledTrackEncodings);
     } else {
       transceiverConfig = {
         direction: "sendonly",
@@ -505,7 +538,6 @@ export class MembraneWebRTC {
         transceiverConfig.sendEncodings![0].maxBitrate = trackContext.maxBandwidth * 1024; // convert to bps;
       }
     }
-
     this.connection!.addTransceiver(track, transceiverConfig);
   };
 
@@ -993,7 +1025,7 @@ export class MembraneWebRTC {
         peer: peer,
         trackId,
         metadata,
-        isSimulcast: false,
+        simulcastConfig: { enabled: false, active_encodings: [] },
       };
 
       this.trackIdToTrack.set(trackId, trackContext);

guides/simulcast.md

@@ -4,29 +4,28 @@ Simulcast is a technique where a client sends multiple encodings of the same vid
 
 * receiver awailable bandwidth
 * receiver preferences (e.g. explicit request to receive video in HD resolution instead of FHD)
-* UI layaout (e.g. videos being displayed in smaller video tiles will be sent in lower resolution)
+* UI layaout (e.g. videos being displayed in smaller video tiles will be sent in a lower resolution)
 
-At the moment, Membrane supports only receiver preferences i.e. receiver can chose which encoding it is willing to receive. Additionaly, sender can turn off/on specific encoding. Membrane RTC Engine will detect changes and switch to another available encoding.Simulcast is a technique where client sends multiple encodings (different resolutions) of the same
-video to a server and the server forwards proper encoding to each other client basing on 
-client preferences, network bandwidth or UI layaout.
+At the moment, Membrane supports only receiver preferences i.e. receiver can chose which encoding it is willing to receive. Additionaly, sender can turn off/on specific encoding. Membrane RTC Engine will detect changes and switch to another available encoding.
 
 ## Turning simulcast on/off
 
-On the client side simulcast can be enabled while adding new track e.g.:
+On the client side simulcast can be enabled while adding a new track e.g.:
 
 ```ts
     // create MembraneWebRTC class instance
     // ...
     // add simulcasted track
-    let trackId = webrtc.addTrack(track, stream, {}, true);
+    let trackId = webrtc.addTrack(track, stream, {}, {enabled: true, active_encodings: ["l", "m", "h"]});
 ```
 
 This will add a new track that will be sent in three versions:
 * original (identified as `h`)
 * original scaled down by 2 (identified as `m`)
 * original scaled down by 4 (identified as `l`)
 
-Those settings are not configurable at the moment.
+You can turn off some of the encodings by excluding them from `active_encodings` list.
+Encodings that are turned off might still be enabled using `enableTrackEncoding` function.
 
 > #### Minimal required resolution {: .warning}
 >
@@ -36,9 +35,24 @@ Those settings are not configurable at the moment.
 > passed to [`getUserMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) 
 > or [`getDisplayMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia).
 
-On the server side, simulcast can be disabled while adding new WebRTC Endpoint by setting its `simulcast?` 
-option to `false`.
-This will result in rejecting all incoming simulcast tracks i.e. client will not send them.
+On the server side, simulcast can be configured while adding new WebRTC Endpoint by setting its `simulcast_config` option.
+
+For example
+
+```elixir
+%WebRTC{
+  rtc_engine: rtc_engine,
+  # ...
+  simulcast_config: %SimulcastConfig{
+    enabled: true,
+    default_encoding: fn %Track{simulcast_encodings: _simulcast_encodings} -> "m" end
+  }
+}
+```
+
+Here we turn simulcast on and choose medium encoding for each track to be forwarded to the client.
+
+On the other hand, setting `enabled` to `false` will result in rejecting all incoming simulcast tracks i.e. client will not send them to the server.
 
 ## Disabling and enabling specific track encoding
 
@@ -58,7 +72,7 @@ Disabled encoding can be turned on again using `enableTrackEncoding` function.
 Membrane RTC Engine tracks encoding activity. 
 Therefore, when some encoding is turned off, RTC Engine will detect this and switch to 
 the highest awailable encoding.
-If turned off encoding returns, RTC Engine will switch back to it.
+When disabled encoding becomes active again, RTC Engine will switch back to it.
 
 ## Selecting encoding to receive
 

lib/membrane_rtc_engine/endpoints/hls_endpoint.ex

@@ -82,23 +82,17 @@ if Enum.all?(
     def handle_other({:new_tracks, tracks}, ctx, state) do
       new_tracks = Map.new(tracks, &{&1.id, &1})
 
-      subscriptions =
-        tracks
-        |> Enum.filter(fn track -> :raw in track.format end)
-        |> Enum.map(fn track -> {track.id, :raw} end)
-
       {:endpoint, endpoint_id} = ctx.name
 
-      case Engine.subscribe(state.rtc_engine, subscriptions, endpoint_id) do
-        :ok ->
-          {:ok, Map.update!(state, :tracks, &Map.merge(&1, new_tracks))}
-
-        {:error, track_id, reason} ->
-          raise "Subscription fails on track: #{track_id} because of #{reason}"
+      Enum.each(tracks, fn track ->
+        case Engine.subscribe(state.rtc_engine, endpoint_id, track.id, :RTP) do
+          :ok ->
+            {:ok, Map.update!(state, :tracks, &Map.merge(&1, new_tracks))}
 
-        {:error, :timeout} ->
-          raise "Timeout subscribing on track in Engine with pid #{state.rtc_engine}"
-      end
+          {:error, reason} ->
+            raise "Couldn't subscribe for track: #{inspect(track.id)}. Reason: #{inspect(reason)}"
+        end
+      end)
     end
 
     @impl true

lib/membrane_rtc_engine/endpoints/webrtc/forwarder.ex

@@ -24,31 +24,52 @@ defmodule Membrane.RTC.Engine.Endpoint.WebRTC.Forwarder do
           old_encoding: String.t() | nil,
           rtp_munger: RTPMunger.t(),
           vp8_munger: VP8Munger.t(),
+          encodings: [String.t()],
           active_encodings: [String.t()],
           started?: boolean()
         }
 
-  @enforce_keys [:codec, :rtp_munger]
+  @enforce_keys [:codec, :selected_encoding, :encodings, :active_encodings, :rtp_munger]
   defstruct @enforce_keys ++
               [
                 :queued_encoding,
                 :old_encoding,
                 :vp8_munger,
-                selected_encoding: "h",
-                active_encodings: ["h", "m", "l"],
                 started?: false
               ]
 
   @doc """
   Creates a new forwarder.
+
+  * `encodings` - a list of possible encodings.
+  * `selected_encoding` - encoding to forward. If not provided,
+  the highest possible encoding will be choosen.
   """
-  @spec new(:H264 | :VP8, Membrane.RTP.clock_rate_t()) :: t()
-  def new(:VP8, clock_rate) do
-    %__MODULE__{codec: :VP8, rtp_munger: RTPMunger.new(clock_rate), vp8_munger: VP8Munger.new()}
+  @spec new(:H264 | :VP8, Membrane.RTP.clock_rate_t(), [String.t()], String.t() | nil) :: t()
+  def new(codec, clock_rate, encodings, selected_encoding \\ nil)
+
+  def new(:VP8, clock_rate, encodings, selected_encoding) do
+    %__MODULE__{
+      codec: :VP8,
+      rtp_munger: RTPMunger.new(clock_rate),
+      vp8_munger: VP8Munger.new(),
+      encodings: encodings,
+      # assume that, at the beginning, all encodings are active
+      # if some encoding is inactive, we will be notified
+      # about this in `encoding_inactive` function
+      active_encodings: encodings,
+      selected_encoding: selected_encoding || get_next_encoding(encodings)
+    }
   end
 
-  def new(:H264, clock_rate) do
-    %__MODULE__{codec: :H264, rtp_munger: RTPMunger.new(clock_rate)}
+  def new(:H264, clock_rate, encodings, selected_encoding) do
+    %__MODULE__{
+      codec: :H264,
+      rtp_munger: RTPMunger.new(clock_rate),
+      encodings: encodings,
+      active_encodings: encodings,
+      selected_encoding: selected_encoding || get_next_encoding(encodings)
+    }
   end
 
   @doc """
@@ -59,26 +80,26 @@ defmodule Membrane.RTC.Engine.Endpoint.WebRTC.Forwarder do
   active again.
   """
   @spec encoding_inactive(t(), String.t()) :: t()
-  def encoding_inactive(
-        %__MODULE__{selected_encoding: encoding, old_encoding: nil} = forwarder,
-        encoding
-      ) do
-    forwarder = %__MODULE__{
-      forwarder
-      | old_encoding: encoding,
-        active_encodings: List.delete(forwarder.active_encodings, encoding)
-    }
-
-    do_select_encoding(forwarder, get_next_encoding(forwarder.active_encodings))
-  end
-
   def encoding_inactive(forwarder, encoding) do
     forwarder = %__MODULE__{
       forwarder
       | active_encodings: List.delete(forwarder.active_encodings, encoding)
     }
 
-    do_select_encoding(forwarder, List.first(forwarder.active_encodings))
+    cond do
+      # if this is currently used and selected encoding
+      forwarder.selected_encoding == encoding and forwarder.old_encoding == nil ->
+        forwarder = %__MODULE__{forwarder | old_encoding: encoding}
+        do_select_encoding(forwarder, get_next_encoding(forwarder.active_encodings))
+
+      # if this is currently used encoding but it wasn't explicitly selected
+      # i.e. we switched to it automatically
+      forwarder.selected_encoding == encoding ->
+        do_select_encoding(forwarder, get_next_encoding(forwarder.active_encodings))
+
+      true ->
+        forwarder
+    end
   end
 
   @doc """
@@ -101,7 +122,13 @@ defmodule Membrane.RTC.Engine.Endpoint.WebRTC.Forwarder do
         forwarder = %__MODULE__{forwarder | old_encoding: nil}
         do_select_encoding(forwarder, encoding)
 
-      # if we don't have any active encoding
+      # if we are waiting for selected encoding to become
+      # active again, try to select a new encoding
+      # it might be better than currently used
+      forwarder.old_encoding != nil ->
+        do_select_encoding(forwarder, get_next_encoding(forwarder.active_encodings))
+
+      # if we didn't have any active encoding
       forwarder.selected_encoding == nil ->
         do_select_encoding(forwarder, get_next_encoding(forwarder.active_encodings))