diff --git a/config/runtime.exs b/config/runtime.exs index a0b801f..a541ef1 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,7 +1,8 @@ import Config config :recording_converter, - compositor_path: System.get_env("COMPOSITOR_PATH") + compositor_path: System.get_env("COMPOSITOR_PATH"), + image_url: System.get_env("IMAGE_URL", "https://cdn-icons-png.flaticon.com/512/149/149071.png") if config_env() != :test do config :recording_converter, diff --git a/lib/compositor_utils.ex b/lib/compositor_utils.ex index 088439b..1122d38 100644 --- a/lib/compositor_utils.ex +++ b/lib/compositor_utils.ex @@ -1,6 +1,8 @@ defmodule RecordingConverter.Compositor do @moduledoc false + @text_margin 10 + @letter_width 12 @output_width 1280 @output_height 720 @video_output_id "video_output_1" @@ -35,35 +37,12 @@ defmodule RecordingConverter.Compositor do @spec video_output_id() :: String.t() def video_output_id(), do: @video_output_id - @spec generate_output_update(String.t(), list(), number()) :: tuple() - def generate_output_update("video", video_tracks, timestamp) when is_list(video_tracks) do - { - :lc_request, - %{ - type: :update_output, - output_id: @video_output_id, - video: - video_tracks - |> Enum.map(&%{type: :input_stream, input_id: &1.id}) - |> scene(), - schedule_time_ms: from_ns_to_ms(timestamp) - } - } - end - - def generate_output_update("audio", audio_tracks, timestamp) when is_list(audio_tracks) do - { - :lc_request, - %{ - type: :update_output, - output_id: @audio_output_id, - audio: %{ - inputs: Enum.map(audio_tracks, &%{input_id: &1.id}) - }, - schedule_time_ms: from_ns_to_ms(timestamp) - } - } - end + @spec generate_output_update(map(), number()) :: [tuple()] + def generate_output_update(tracks, timestamp), + do: [ + generate_video_output_update(tracks, timestamp), + generate_audio_output_update(tracks, timestamp) + ] @spec schedule_unregister_video_output(number()) :: {:lc_request, map()} def schedule_unregister_video_output(schedule_time_ns), @@ -101,6 +80,121 @@ defmodule RecordingConverter.Compositor do } } + @spec register_image_action(String.t()) :: {:lc_request, map()} + def register_image_action(image_url) do + { + :lc_request, + %{ + type: "register", + entity_type: "image", + asset_type: "png", + image_id: "avatar_png", + url: image_url + } + } + end + + defp generate_video_output_update( + %{"video" => video_tracks, "audio" => audio_tracks}, + timestamp + ) + when is_list(video_tracks) do + video_tracks_id = Enum.map(video_tracks, fn track -> track["origin"] end) + avatar_tracks = Enum.reject(audio_tracks, fn track -> track["origin"] in video_tracks_id end) + + avatars_config = Enum.map(avatar_tracks, &avatar_view/1) + video_tracks_config = Enum.map(video_tracks, &video_input_source_view/1) + + { + :lc_request, + %{ + type: :update_output, + output_id: @video_output_id, + schedule_time_ms: from_ns_to_ms(timestamp), + video: scene(video_tracks_config ++ avatars_config) + } + } + end + + defp generate_audio_output_update(%{"audio" => audio_tracks}, timestamp) + when is_list(audio_tracks) do + { + :lc_request, + %{ + type: :update_output, + output_id: @audio_output_id, + audio: %{ + inputs: Enum.map(audio_tracks, &%{input_id: &1.id}) + }, + schedule_time_ms: from_ns_to_ms(timestamp) + } + } + end + + defp video_input_source_view(track) do + %{ + type: :view, + children: + [ + # TODO: fix after compositor update + # unnecessary rescaler + %{ + type: "rescaler", + mode: "fit", + child: %{ + type: :input_stream, + input_id: track.id + } + } + ] ++ text_view(track["metadata"]) + } + end + + defp avatar_view(track) do + %{ + type: "view", + children: + [ + # TODO: fix after compositor update + # unnecessary rescaler + %{ + type: "rescaler", + mode: "fit", + child: %{ + type: "image", + image_id: "avatar_png" + } + } + ] ++ text_view(track["metadata"]) + } + end + + defp text_view(%{"displayName" => label}) do + label_width = String.length(label) * @letter_width + @text_margin + + [ + %{ + type: "view", + bottom: 20, + right: 20, + width: label_width, + height: 20, + background_color_rgba: "#000000FF", + children: [ + %{ + type: "text", + text: label, + align: "center", + width: label_width, + font_size: 20.0 + } + ] + } + ] + end + + defp text_view(_metadata), do: [] + defp from_ns_to_ms(timestamp_ns) do rounded_ts = timestamp_ns |> Membrane.Time.nanoseconds() |> Membrane.Time.as_milliseconds(:round) diff --git a/lib/pipeline.ex b/lib/pipeline.ex index da3cb8b..7b0a60c 100644 --- a/lib/pipeline.ex +++ b/lib/pipeline.ex @@ -52,12 +52,15 @@ defmodule RecordingConverter.Pipeline do tracks_spec = Enum.map(tracks, &create_branch(&1, state)) - actions = + track_actions = tracks |> ReportParser.get_all_track_actions() |> Enum.map(¬ify_compositor/1) - actions = [{:spec, tracks_spec} | actions] + register_image_action = + state.image_url |> Compositor.register_image_action() |> notify_compositor() + + actions = [{:spec, tracks_spec}, register_image_action | track_actions] {actions, %{ diff --git a/lib/recording_converter.ex b/lib/recording_converter.ex index d2f6a9f..474b7fa 100644 --- a/lib/recording_converter.ex +++ b/lib/recording_converter.ex @@ -29,7 +29,8 @@ defmodule RecordingConverter do bucket_name: bucket_name(), compositor_path: compositor_path(), s3_directory: s3_directory(), - output_directory: output_directory() + output_directory: output_directory(), + image_url: image_url() }) Process.monitor(pipeline_pid) @@ -130,6 +131,10 @@ defmodule RecordingConverter do end end + defp image_url() do + Application.fetch_env!(:recording_converter, :image_url) + end + defp convert_to_absolute_path(output_directory) do s3_directory() |> Path.join(output_directory) diff --git a/lib/report_parser.ex b/lib/report_parser.ex index b784133..792e723 100644 --- a/lib/report_parser.ex +++ b/lib/report_parser.ex @@ -51,13 +51,14 @@ defmodule RecordingConverter.ReportParser do |> Enum.map_reduce(%{"audio" => [], "video" => []}, fn {:start, %{"type" => type} = track, timestamp}, acc -> acc = Map.update!(acc, type, &[track | &1]) - {Compositor.generate_output_update(type, acc[type], timestamp), acc} + {Compositor.generate_output_update(acc, timestamp), acc} {:end, %{"type" => type} = track, timestamp}, acc -> acc = Map.update!(acc, type, fn tracks -> Enum.reject(tracks, &(&1 == track)) end) - {Compositor.generate_output_update(type, acc[type], timestamp), acc} + {Compositor.generate_output_update(acc, timestamp), acc} end) |> then(fn {actions, _acc} -> actions end) + |> List.flatten() end defp generate_unregister_output_actions(track_actions) do diff --git a/mix.lock b/mix.lock index 079d913..d03f5d0 100644 --- a/mix.lock +++ b/mix.lock @@ -15,7 +15,7 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "ex_aws": {:hex, :ex_aws, "2.5.3", "9c2d05ba0c057395b12c7b5ca6267d14cdaec1d8e65bdf6481fe1fd245accfb4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67115f1d399d7ec4d191812ee565c6106cb4b1bbf19a9d4db06f265fd87da97e"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"}, - "ex_doc": {:hex, :ex_doc, "0.32.0", "896afb57b1e00030f6ec8b2e19d3ca99a197afb23858d49d94aea673dc222f12", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "ed2c3e42c558f49bda3ff37e05713432006e1719a6c4a3320c7e4735787374e7"}, + "ex_doc": {:hex, :ex_doc, "0.32.1", "21e40f939515373bcdc9cffe65f3b3543f05015ac6c3d01d991874129d173420", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5142c9db521f106d61ff33250f779807ed2a88620e472ac95dc7d59c380113da"}, "ex_sdp": {:hex, :ex_sdp, "0.15.0", "53815fb5b5e4fae0f3b26de90f372446bb8e0eed62a3cc20394d3c29519698be", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "d3f23596b73e7057521ff0f0d55b1189c6320a2f04388aa3a80a0aa97ffb379f"}, "excoveralls": {:hex, :excoveralls, "0.15.3", "54bb54043e1cf5fe431eb3db36b25e8fd62cf3976666bafe491e3fa5e29eba47", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f8eb5d8134d84c327685f7bb8f1db4147f1363c3c9533928234e496e3070114e"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, @@ -29,7 +29,7 @@ "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"}, - "membrane_aac_fdk_plugin": {:hex, :membrane_aac_fdk_plugin, "0.18.7", "4d9af018c22d9291b72d6025941452dd53c7921bcdbc826da8866bb6ecefa8cb", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "79904c3b78882bd0cec15b02928e6b53780602e64a359941acbc9a2408e7b74b"}, + "membrane_aac_fdk_plugin": {:hex, :membrane_aac_fdk_plugin, "0.18.8", "88d47923805cbd9a977fc7e5d3eb8d3028a2e358ad9ad7b124684adc78c2e8ee", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "bb9e706d0949954affd4e295f5d3d4660096997756b5422119800d961c46cc63"}, "membrane_aac_format": {:hex, :membrane_aac_format, "0.8.0", "515631eabd6e584e0e9af2cea80471fee6246484dbbefc4726c1d93ece8e0838", [:mix], [{:bimap, "~> 1.1", [hex: :bimap, repo: "hexpm", optional: false]}], "hexpm", "a30176a94491033ed32be45e51d509fc70a5ee6e751f12fd6c0d60bd637013f6"}, "membrane_aac_plugin": {:hex, :membrane_aac_plugin, "0.18.1", "30433bffd4d5d773f79448dd9afd55d77338721688f09a89b20d742a68cc2c3d", [:mix], [{:bunch, "~> 1.0", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "8fd048c47d5d2949eb557e19f43f62d534d3af5096187f1a1a3a1694d14b772c"}, "membrane_aws_plugin": {:git, "https://github.com/jellyfish-dev/membrane_aws_plugin.git", "41b59a7e564305436da89fb753a1d14edc161af5", []}, diff --git a/test/fixtures/multiple-audios-and-one-video/a89421bc-8466-485d-ad52-7f574acc4084.msr b/test/fixtures/multiple-audios-and-one-video/a89421bc-8466-485d-ad52-7f574acc4084.msr new file mode 100644 index 0000000..c20408c Binary files /dev/null and b/test/fixtures/multiple-audios-and-one-video/a89421bc-8466-485d-ad52-7f574acc4084.msr differ diff --git a/test/fixtures/multiple-audios-and-one-video/b96043e6-abaa-40bd-9267-12794f6e5529.msr b/test/fixtures/multiple-audios-and-one-video/b96043e6-abaa-40bd-9267-12794f6e5529.msr new file mode 100644 index 0000000..8842513 Binary files /dev/null and b/test/fixtures/multiple-audios-and-one-video/b96043e6-abaa-40bd-9267-12794f6e5529.msr differ diff --git a/test/fixtures/multiple-audios-and-one-video/dd78e5bd-aa5a-4e01-abbc-354227ab7529.msr b/test/fixtures/multiple-audios-and-one-video/dd78e5bd-aa5a-4e01-abbc-354227ab7529.msr new file mode 100644 index 0000000..311b3af Binary files /dev/null and b/test/fixtures/multiple-audios-and-one-video/dd78e5bd-aa5a-4e01-abbc-354227ab7529.msr differ diff --git a/test/fixtures/multiple-audios-and-one-video/report.json b/test/fixtures/multiple-audios-and-one-video/report.json new file mode 100644 index 0000000..7ba6172 --- /dev/null +++ b/test/fixtures/multiple-audios-and-one-video/report.json @@ -0,0 +1,40 @@ +{ + "tracks": { + + "a89421bc-8466-485d-ad52-7f574acc4084.msr": { + "offset": 18052166, + "type": "audio", + "encoding": "OPUS", + "metadata": {"displayName": "Avatar"}, + "clock_rate": 48000, + "start_timestamp": 775938176, + "end_timestamp": 776421056, + "origin": 1 + }, + "b96043e6-abaa-40bd-9267-12794f6e5529.msr": { + "offset": 0, + "type": "video", + "encoding": "H264", + "metadata": { + "isScreenSharing": false, + "mainPresenter": true, + "displayName": "really long username" + }, + "clock_rate": 90000, + "start_timestamp": 3637718073, + "end_timestamp": 3638616573, + "origin": 2 + }, + "dd78e5bd-aa5a-4e01-abbc-354227ab7529.msr": { + "offset": 12473666, + "type": "audio", + "encoding": "OPUS", + "metadata": null , + "clock_rate": 48000, + "start_timestamp": 2095834478, + "end_timestamp": 2096317358, + "origin": 2 + } + }, + "recording_id": "recording_id" +} \ No newline at end of file diff --git a/test/pipeline_test.exs b/test/pipeline_test.exs index 71b5815..9030475 100644 --- a/test/pipeline_test.exs +++ b/test/pipeline_test.exs @@ -49,6 +49,7 @@ defmodule RecordingConverter.PipelineTest do %{type: "one-audio", requests: 4, factor: 1}, %{type: "one-video", requests: 8, factor: 1}, %{type: "multiple-audios-and-videos", requests: 18, factor: 1}, + %{type: "multiple-audios-and-one-video", requests: 12, factor: 1}, %{type: "long-video", requests: 16, factor: 5} ] @@ -182,7 +183,8 @@ defmodule RecordingConverter.PipelineTest do do: "test_path/output", else: "output/" ), - compositor_path: state.compositor_path + compositor_path: state.compositor_path, + image_url: Application.fetch_env!(:recording_converter, :image_url) } )