[RTC-230] Complete packet loss test (#292)

Author: sgfn

integration_test/README.md

@@ -13,4 +13,4 @@ Simple test using four containers: `test_videoroom` and three `test_browser`s. T
 to videoroom, start sending and receiving media, then we apply packet loss on one of the browser
 containers and verify that other browsers receive less frames only from this particular one.
 
-To run, execute `./test_packet_loss.sh`
+To run, execute `./run_packet_loss_test.sh`

integration_test/docker-compose.yml

@@ -25,7 +25,6 @@ services:
       USE_TLS: "true"
       KEY_FILE_PATH: "priv/cert/selfsigned_key.pem"
       CERT_FILE_PATH: "priv/cert/selfsigned.pem"
-      USE_RESULT_RECEIVER: "true"
     networks:
       network:
         ipv4_address: 192.168.0.50

integration_test/test_packet_loss.sh integration_test/run_packet_loss_test.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+TEST_DEPS="docker pumba"
+
 # test phase durations (in seconds)
 LOSS_DURATION=60
 
@@ -12,9 +14,17 @@ set -e
 
 rm -rf $SHARED_VOLUME_DIR
 
+for dep in $TEST_DEPS; do
+  if [ ! $(which $dep) ]; then
+    echo "Unable to run test (missing dependency: $dep)"
+    exit 1
+  fi
+done
+
 docker compose build
 echo "Running packet loss test"
 docker compose up --exit-code-from=server server browser0 browser1 browser2 &
+COMPOSE_JOB_ID=$!
 
 cleanup() {
   docker compose down --rmi local --volumes
@@ -32,18 +42,15 @@ while [ ! -f "${SHARED_VOLUME_DIR}/ENABLE_PACKET_LOSS" ]; do
   sleep 1
 done
 
-# The netem command will return an error when a container is stopped before the packet loss duration
-# is up. This means we either need to kill it (and know when to do that), or ignore the error:
-set +e
-
 echo "Applying packet loss to $APPLY_LOSS_TO for $LOSS_DURATION seconds"
 pumba netem \
   --duration "${LOSS_DURATION}s" \
   loss \
   --percent 50 \
-  $APPLY_LOSS_TO
+  $APPLY_LOSS_TO &
 
-echo "Network condition simulation over. Waiting for the docker-compose job to complete..."
+NETEM_JOB_ID=$!
+trap "set +e; kill $NETEM_JOB_ID; cleanup" EXIT
 
-wait
+wait $COMPOSE_JOB_ID
 exit $?

integration_test/test_browser/lib/test_browser/mustang.ex

@@ -6,7 +6,10 @@ defmodule TestBrowser.Mustang do
   def join(browser, options) do
     page = browser |> Playwright.Browser.new_page()
     _response = Playwright.Page.goto(page, options.target_url)
-    Playwright.Page.on(page, :console, fn msg -> Logger.info("Browser console: #{msg.params.message.message_text}") end)
+
+    Playwright.Page.on(page, :console, fn msg ->
+      Logger.info("Browser console: #{msg.params.message.message_text}")
+    end)
 
     page
     |> Playwright.Page.locator("[id=#{options.start_button}]")
@@ -46,7 +49,6 @@ defmodule TestBrowser.Mustang do
       {:notify_server, msg} = action ->
         Logger.info("mustang: #{options.id}, action: #{inspect(action)}")
         send(options.server, {msg, options.id})
-
     end)
 
     ctx

integration_test/test_browser/mix.exs

@@ -25,7 +25,7 @@ defmodule TestBrowser.MixProject do
   defp deps do
     [
       {:cowlib, "~> 2.11", override: true},
-      {:stampede, github: "membraneframework-labs/stampede-elixir"},
+      {:stampede, github: "membraneframework-labs/stampede-elixir"}
     ]
   end
 end

integration_test/test_browser/test/test_browser_test.exs

@@ -23,7 +23,10 @@ defmodule TestBrowserTest do
     Process.sleep(@test_warmup_time)
 
     server = System.get_env("SERVER_HOSTNAME", "localhost")
-    server_receiver = {TestVideoroom.TestResultReceiver, String.to_atom(server <> "@" <> server)}
+
+    server_receiver =
+      {TestVideoroom.Integration.ResultReceiver, String.to_atom(server <> "@" <> server)}
+
     stats_task = Task.async(fn -> receive_stats(server_receiver) end)
 
     hostname = with {:ok, hostname} <- :inet.gethostname(), do: to_string(hostname)
@@ -35,14 +38,15 @@ defmodule TestBrowserTest do
       receiver: stats_task.pid,
       server: server_receiver,
       actions: [
-        {:get_stats, @simulcast_inbound_stats, @stats_number, @stats_interval, tag: :after_warmup},
+        {:get_stats, @simulcast_inbound_stats, @stats_number, @stats_interval,
+         tag: :after_warmup},
         {:wait, @packet_loss_warmup_time},
         {:notify_server, :enable_packet_loss},
         {:wait, @packet_loss_warmup_time},
         {:get_stats, @simulcast_inbound_stats, @stats_number, @stats_interval,
-          tag: :after_applying_packet_loss_on_one_user},
+         tag: :after_applying_packet_loss_on_one_user},
 
-        # Get local stream stats so that we know our stream ID
+        # Get local stream stats so that we know our peer ID
         {:get_stats, @simulcast_outbound_stats, 1, 0, tag: :local_stream_stats}
       ],
       id: hostname

integration_test/test_videoroom/Dockerfile

@@ -34,6 +34,8 @@ RUN mix deps.compile
 COPY ${APP_PATH}/lib lib/
 RUN mix compile
 
+COPY ${APP_PATH}/test test/
+
 COPY ${APP_PATH}/config/runtime.exs config/
 
 COPY ${APP_PATH}/assets assets/

integration_test/test_videoroom/assets/js/app.js

@@ -108,8 +108,7 @@ async function refreshStats(statsFunction) {
       .webrtc.connection}`;
     return;
   }
-  // we are accessing room's private field, in the name of science of course...
-  const stats = await statsFunction(room.webrtc.connection);
+  const stats = await statsFunction(room);
 
   putStats(stats)
 }
@@ -147,11 +146,5 @@ localHighEncodingButton.onclick = () => { toggleSimulcastEncoding(localHighEncod
 peerLowEncodingButton.onclick = () => { room.selectPeerSimulcastEncoding("l") }
 peerMediumEncodingButton.onclick = () => { room.selectPeerSimulcastEncoding("m") }
 peerHighEncodingButton.onclick = () => { room.selectPeerSimulcastEncoding("h") }
-inboundSimulcastStatsButton.onclick = () => {
-  refreshStats(async (connection) => {
-    let stats = await inboundSimulcastStreamStats(connection)
-    stats.encoding = room.getPeerEncoding()
-    return stats
-  })
-}
+inboundSimulcastStatsButton.onclick = () => { refreshStats(inboundSimulcastStreamStats) }
 outboundSimulcastStatsButton.onclick = () => { refreshStats(outboundSimulcastStreamStats) }

integration_test/test_videoroom/assets/js/room.js

@@ -35,6 +35,7 @@ class Room {
     this.encodings = ["l", "m", "h"];
     this.peerMetadata = null;
     this.trackMetadata = null;
+    this.peerIdToVideoTrack = {};
     this.selfId = null;
     this.simulcast = simulcast;
     this.remoteTracks = new Map();
@@ -71,6 +72,10 @@ class Room {
 
       video.srcObject = ctx.stream;
       this.remoteTracks.set(ctx.trackId, ctx);
+
+      if (ctx.track.kind === "video") {
+        this.peerIdToVideoTrack[ctx.endpoint.id] = ctx.track;
+      }
     });
 
     this.webrtc.on("trackAdded", (ctx) => {

integration_test/test_videoroom/assets/js/stats.js

@@ -94,36 +94,47 @@ async function isAudioPlayingFirefox(peerConnection, audioTrack) {
   return packetsStart > 0 && packetsEnd > 0 && packetsEnd > packetsStart;
 }
 
-export async function inboundSimulcastStreamStats(peerConnection) {
-  const stats = await peerConnection.getStats();
-  let data = { height: null, width: null, framesPerSecond: 0 }
-  for (let [_key, report] of stats) {
-    if (report.type == "inbound-rtp") {
-      data = getDataFromReport(report)
-      data.framesReceived = report.framesReceived
+export async function inboundSimulcastStreamStats(room) {
+  // we are accessing room's private fields, in the name of science of course...
+  const peerConnection = room.webrtc.connection;
+
+  const stats = room.peers.map(async (peer) => {
+    const videoTrack = room.peerIdToVideoTrack[peer.id];
+    const track_stats = await peerConnection.getStats(videoTrack);
+
+    let data = { peerId: peer.id, height: null, width: null, framesPerSecond: 0 }
+    for (let [_key, report] of track_stats) {
+      if (report.type == "inbound-rtp") {
+        data = getDataFromReport(report)
+        data.framesReceived = report.framesReceived
+        data.peerId = peer.id
+        // The following works for one peer only...
+        data.encoding = room.getPeerEncoding()
+      }
     }
-  }
-
-  return data
-}
 
+    return data;
+  });
 
+  return (await Promise.all(stats));
+}
 
-export async function outboundSimulcastStreamStats(peerConnection) {
+export async function outboundSimulcastStreamStats(room) {
+  const peerConnection = room.webrtc.connection;
   const stats = await peerConnection.getStats();
 
-  let streams = { "l": null, "m": null, "h": null }
+  let data = { peerId: room.selfId, "l": null, "m": null, "h": null }
   for (let [_key, report] of stats) {
     if (report.type == "outbound-rtp") {
       let rid = report.rid
-      streams[rid] = getDataFromReport(report)
-      streams[rid].framesSent = report.framesSent
-      streams[rid].qualityLimitationDuration = report["qualityLimitationDurations"]
-      streams[rid].qualityLimitationReason = report["qualityLimitationReason"]
+      data[rid] = getDataFromReport(report)
+      data[rid].framesSent = report.framesSent
+      data[rid].qualityLimitationDuration = report["qualityLimitationDurations"]
+      data[rid].qualityLimitationReason = report["qualityLimitationReason"]
     }
   }
 
-  return streams
+  return data
 }
 
 function getDataFromReport(values) {
@@ -134,7 +145,8 @@ function getDataFromReport(values) {
   return data
 }
 
-export async function remoteStreamsStats(peerConnection) {
+export async function remoteStreamsStats(room) {
+  const peerConnection = room.webrtc.connection;
   const streams = peerConnection.getRemoteStreams();
 
   const firefoxTrackActive = peerConnection

integration_test/test_videoroom/config/runtime.exs

@@ -56,8 +56,7 @@ config :test_videoroom,
     |> ConfigParser.parse_port_number("INTEGRATED_TLS_TURN_PORT"),
   integrated_turn_pkey: System.get_env("INTEGRATED_TURN_PKEY"),
   integrated_turn_cert: System.get_env("INTEGRATED_TURN_CERT"),
-  integrated_turn_domain: System.get_env("VIRTUAL_HOST"),
-  use_result_receiver: System.get_env("USE_RESULT_RECEIVER") == "true"
+  integrated_turn_domain: System.get_env("VIRTUAL_HOST")
 
 protocol = if System.get_env("USE_TLS") == "true", do: :https, else: :http
 

integration_test/test_videoroom/docker-entrypoint.sh

@@ -4,5 +4,5 @@
 if [ $# -gt 0 ]; then
   exec "$@"
 else
-  elixir --sname $(hostname) --cookie "$ERL_COOKIE" -S mix run --no-halt
+  elixir --sname $(hostname) --cookie "$ERL_COOKIE" -S mix test.containerised --only packet_loss_test
 fi

integration_test/test_videoroom/lib/test_videoroom/application.ex

@@ -10,11 +10,6 @@ defmodule TestVideoroom.Application do
       TestVideoroomWeb.Endpoint
     ]
 
-    children =
-      if Application.fetch_env!(:test_videoroom, :use_result_receiver),
-        do: children ++ [{TestVideoroom.TestResultReceiver, restart: :temporary}],
-        else: children
-
     opts = [strategy: :one_for_one, name: TestVideoroom.Supervisor]
 
     Supervisor.start_link(children, opts)

integration_test/test_videoroom/lib/test_videoroom/test_result_receiver.ex

@@ -42,8 +42,9 @@ defmodule TestVideoroom.MixProject do
 
   defp aliases() do
     [
-      test: ["assets.deploy", "test"],
-      "assets.deploy": ["esbuild default --minify", "phx.digest"]
+      test: ["assets.deploy", "test --exclude containerised"],
+      "assets.deploy": ["esbuild default --minify", "phx.digest"],
+      "test.containerised": ["test --only containerised"]
     ]
   end
 end