Merge pull request #3 from jellyfish-dev/add-avatars-and-display-name

Add avatars and display names

config/runtime.exs

@@ -1,7 +1,8 @@
 import Config
 
 config :recording_converter,
-  compositor_path: System.get_env("COMPOSITOR_PATH")
+  compositor_path: System.get_env("COMPOSITOR_PATH"),
+  image_url: System.get_env("IMAGE_URL", "https://cdn-icons-png.flaticon.com/512/149/149071.png")
 
 if config_env() != :test do
   config :recording_converter,

lib/compositor_utils.ex

@@ -1,6 +1,8 @@
 defmodule RecordingConverter.Compositor do
   @moduledoc false
 
+  @text_margin 10
+  @letter_width 12
   @output_width 1280
   @output_height 720
   @video_output_id "video_output_1"
@@ -35,35 +37,12 @@ defmodule RecordingConverter.Compositor do
   @spec video_output_id() :: String.t()
   def video_output_id(), do: @video_output_id
 
-  @spec generate_output_update(String.t(), list(), number()) :: tuple()
-  def generate_output_update("video", video_tracks, timestamp) when is_list(video_tracks) do
-    {
-      :lc_request,
-      %{
-        type: :update_output,
-        output_id: @video_output_id,
-        video:
-          video_tracks
-          |> Enum.map(&%{type: :input_stream, input_id: &1.id})
-          |> scene(),
-        schedule_time_ms: from_ns_to_ms(timestamp)
-      }
-    }
-  end
-
-  def generate_output_update("audio", audio_tracks, timestamp) when is_list(audio_tracks) do
-    {
-      :lc_request,
-      %{
-        type: :update_output,
-        output_id: @audio_output_id,
-        audio: %{
-          inputs: Enum.map(audio_tracks, &%{input_id: &1.id})
-        },
-        schedule_time_ms: from_ns_to_ms(timestamp)
-      }
-    }
-  end
+  @spec generate_output_update(map(), number()) :: [tuple()]
+  def generate_output_update(tracks, timestamp),
+    do: [
+      generate_video_output_update(tracks, timestamp),
+      generate_audio_output_update(tracks, timestamp)
+    ]
 
   @spec schedule_unregister_video_output(number()) :: {:lc_request, map()}
   def schedule_unregister_video_output(schedule_time_ns),
@@ -101,6 +80,121 @@ defmodule RecordingConverter.Compositor do
       }
     }
 
+  @spec register_image_action(String.t()) :: {:lc_request, map()}
+  def register_image_action(image_url) do
+    {
+      :lc_request,
+      %{
+        type: "register",
+        entity_type: "image",
+        asset_type: "png",
+        image_id: "avatar_png",
+        url: image_url
+      }
+    }
+  end
+
+  defp generate_video_output_update(
+         %{"video" => video_tracks, "audio" => audio_tracks},
+         timestamp
+       )
+       when is_list(video_tracks) do
+    video_tracks_id = Enum.map(video_tracks, fn track -> track["origin"] end)
+    avatar_tracks = Enum.reject(audio_tracks, fn track -> track["origin"] in video_tracks_id end)
+
+    avatars_config = Enum.map(avatar_tracks, &avatar_view/1)
+    video_tracks_config = Enum.map(video_tracks, &video_input_source_view/1)
+
+    {
+      :lc_request,
+      %{
+        type: :update_output,
+        output_id: @video_output_id,
+        schedule_time_ms: from_ns_to_ms(timestamp),
+        video: scene(video_tracks_config ++ avatars_config)
+      }
+    }
+  end
+
+  defp generate_audio_output_update(%{"audio" => audio_tracks}, timestamp)
+       when is_list(audio_tracks) do
+    {
+      :lc_request,
+      %{
+        type: :update_output,
+        output_id: @audio_output_id,
+        audio: %{
+          inputs: Enum.map(audio_tracks, &%{input_id: &1.id})
+        },
+        schedule_time_ms: from_ns_to_ms(timestamp)
+      }
+    }
+  end
+
+  defp video_input_source_view(track) do
+    %{
+      type: :view,
+      children:
+        [
+          # TODO: fix after compositor update
+          # unnecessary rescaler
+          %{
+            type: "rescaler",
+            mode: "fit",
+            child: %{
+              type: :input_stream,
+              input_id: track.id
+            }
+          }
+        ] ++ text_view(track["metadata"])
+    }
+  end
+
+  defp avatar_view(track) do
+    %{
+      type: "view",
+      children:
+        [
+          # TODO: fix after compositor update
+          # unnecessary rescaler
+          %{
+            type: "rescaler",
+            mode: "fit",
+            child: %{
+              type: "image",
+              image_id: "avatar_png"
+            }
+          }
+        ] ++ text_view(track["metadata"])
+    }
+  end
+
+  defp text_view(%{"displayName" => label}) do
+    label_width = String.length(label) * @letter_width + @text_margin
+
+    [
+      %{
+        type: "view",
+        bottom: 20,
+        right: 20,
+        width: label_width,
+        height: 20,
+        background_color_rgba: "#000000FF",
+        children: [
+          %{
+            type: "text",
+            text: label,
+            align: "center",
+            width: label_width,
+            font_size: 20.0
+          }
+        ]
+      }
+    ]
+  end
+
+  defp text_view(_metadata), do: []
+
   defp from_ns_to_ms(timestamp_ns) do
     rounded_ts =
       timestamp_ns |> Membrane.Time.nanoseconds() |> Membrane.Time.as_milliseconds(:round)

lib/pipeline.ex

@@ -52,12 +52,15 @@ defmodule RecordingConverter.Pipeline do
 
     tracks_spec = Enum.map(tracks, &create_branch(&1, state))
 
-    actions =
+    track_actions =
       tracks
       |> ReportParser.get_all_track_actions()
       |> Enum.map(&notify_compositor/1)
 
-    actions = [{:spec, tracks_spec} | actions]
+    register_image_action =
+      state.image_url |> Compositor.register_image_action() |> notify_compositor()
+
+    actions = [{:spec, tracks_spec}, register_image_action | track_actions]
 
     {actions,
      %{

lib/recording_converter.ex

@@ -29,7 +29,8 @@ defmodule RecordingConverter do
         bucket_name: bucket_name(),
         compositor_path: compositor_path(),
         s3_directory: s3_directory(),
-        output_directory: output_directory()
+        output_directory: output_directory(),
+        image_url: image_url()
       })
 
     Process.monitor(pipeline_pid)
@@ -130,6 +131,10 @@ defmodule RecordingConverter do
     end
   end
 
+  defp image_url() do
+    Application.fetch_env!(:recording_converter, :image_url)
+  end
+
   defp convert_to_absolute_path(output_directory) do
     s3_directory()
     |> Path.join(output_directory)

lib/report_parser.ex

@@ -51,13 +51,14 @@ defmodule RecordingConverter.ReportParser do
     |> Enum.map_reduce(%{"audio" => [], "video" => []}, fn
       {:start, %{"type" => type} = track, timestamp}, acc ->
         acc = Map.update!(acc, type, &[track | &1])
-        {Compositor.generate_output_update(type, acc[type], timestamp), acc}
+        {Compositor.generate_output_update(acc, timestamp), acc}
 
       {:end, %{"type" => type} = track, timestamp}, acc ->
         acc = Map.update!(acc, type, fn tracks -> Enum.reject(tracks, &(&1 == track)) end)
-        {Compositor.generate_output_update(type, acc[type], timestamp), acc}
+        {Compositor.generate_output_update(acc, timestamp), acc}
     end)
     |> then(fn {actions, _acc} -> actions end)
+    |> List.flatten()
   end
 
   defp generate_unregister_output_actions(track_actions) do

mix.lock

@@ -15,7 +15,7 @@
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
   "ex_aws": {:hex, :ex_aws, "2.5.3", "9c2d05ba0c057395b12c7b5ca6267d14cdaec1d8e65bdf6481fe1fd245accfb4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67115f1d399d7ec4d191812ee565c6106cb4b1bbf19a9d4db06f265fd87da97e"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"},
-  "ex_doc": {:hex, :ex_doc, "0.32.0", "896afb57b1e00030f6ec8b2e19d3ca99a197afb23858d49d94aea673dc222f12", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "ed2c3e42c558f49bda3ff37e05713432006e1719a6c4a3320c7e4735787374e7"},
+  "ex_doc": {:hex, :ex_doc, "0.32.1", "21e40f939515373bcdc9cffe65f3b3543f05015ac6c3d01d991874129d173420", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5142c9db521f106d61ff33250f779807ed2a88620e472ac95dc7d59c380113da"},
   "ex_sdp": {:hex, :ex_sdp, "0.15.0", "53815fb5b5e4fae0f3b26de90f372446bb8e0eed62a3cc20394d3c29519698be", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "d3f23596b73e7057521ff0f0d55b1189c6320a2f04388aa3a80a0aa97ffb379f"},
   "excoveralls": {:hex, :excoveralls, "0.15.3", "54bb54043e1cf5fe431eb3db36b25e8fd62cf3976666bafe491e3fa5e29eba47", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f8eb5d8134d84c327685f7bb8f1db4147f1363c3c9533928234e496e3070114e"},
   "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
@@ -29,7 +29,7 @@
   "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
   "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"},
-  "membrane_aac_fdk_plugin": {:hex, :membrane_aac_fdk_plugin, "0.18.7", "4d9af018c22d9291b72d6025941452dd53c7921bcdbc826da8866bb6ecefa8cb", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "79904c3b78882bd0cec15b02928e6b53780602e64a359941acbc9a2408e7b74b"},
+  "membrane_aac_fdk_plugin": {:hex, :membrane_aac_fdk_plugin, "0.18.8", "88d47923805cbd9a977fc7e5d3eb8d3028a2e358ad9ad7b124684adc78c2e8ee", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "bb9e706d0949954affd4e295f5d3d4660096997756b5422119800d961c46cc63"},
   "membrane_aac_format": {:hex, :membrane_aac_format, "0.8.0", "515631eabd6e584e0e9af2cea80471fee6246484dbbefc4726c1d93ece8e0838", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}], "hexpm", "a30176a94491033ed32be45e51d509fc70a5ee6e751f12fd6c0d60bd637013f6"},
   "membrane_aac_plugin": {:hex, :membrane_aac_plugin, "0.18.1", "30433bffd4d5d773f79448dd9afd55d77338721688f09a89b20d742a68cc2c3d", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "8fd048c47d5d2949eb557e19f43f62d534d3af5096187f1a1a3a1694d14b772c"},
   "membrane_aws_plugin": {:git, "https://github.com/jellyfish-dev/membrane_aws_plugin.git", "41b59a7e564305436da89fb753a1d14edc161af5", []},

test/fixtures/multiple-audios-and-one-video/report.json

@@ -0,0 +1,40 @@
+{
+  "tracks": {
+
+    "a89421bc-8466-485d-ad52-7f574acc4084.msr": {
+      "offset": 18052166,
+      "type": "audio",
+      "encoding": "OPUS",
+      "metadata": {"displayName": "Avatar"},
+      "clock_rate": 48000,
+      "start_timestamp": 775938176,
+      "end_timestamp": 776421056,
+      "origin": 1
+    },
+    "b96043e6-abaa-40bd-9267-12794f6e5529.msr": {
+      "offset": 0,
+      "type": "video",
+      "encoding": "H264",
+      "metadata": {
+        "isScreenSharing": false,
+        "mainPresenter": true,
+        "displayName": "really long username"
+      },
+      "clock_rate": 90000,
+      "start_timestamp": 3637718073,
+      "end_timestamp": 3638616573,
+      "origin": 2
+    },
+    "dd78e5bd-aa5a-4e01-abbc-354227ab7529.msr": {
+      "offset": 12473666,
+      "type": "audio",
+      "encoding": "OPUS",
+      "metadata": null ,
+      "clock_rate": 48000,
+      "start_timestamp": 2095834478,
+      "end_timestamp": 2096317358,
+      "origin": 2
+    }
+  },
+  "recording_id": "recording_id"
+}

test/pipeline_test.exs

@@ -49,6 +49,7 @@ defmodule RecordingConverter.PipelineTest do
     %{type: "one-audio", requests: 4, factor: 1},
     %{type: "one-video", requests: 8, factor: 1},
     %{type: "multiple-audios-and-videos", requests: 18, factor: 1},
+    %{type: "multiple-audios-and-one-video", requests: 12, factor: 1},
     %{type: "long-video", requests: 16, factor: 5}
   ]
 
@@ -182,7 +183,8 @@ defmodule RecordingConverter.PipelineTest do
                      do: "test_path/output",
                      else: "output/"
                    ),
-                 compositor_path: state.compositor_path
+                 compositor_path: state.compositor_path,
+                 image_url: Application.fetch_env!(:recording_converter, :image_url)
                }
              )