diff --git a/.gitignore b/.gitignore index 324f6fa..2c94ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -177,3 +177,6 @@ $RECYCLE.BIN/ *.lnk # End of https://www.gitignore.io/api/c,vim,linux,macos,elixir,windows,visualstudiocode + +# Test results default filename +results.csv diff --git a/README.md b/README.md index cee4090..b618d4c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # Jellygrinder -TODO +Utility for running HLS and LL-HLS stress-tests against [the Jellyfish Media Server](https://github.com/jellyfish-dev/jellyfish) ## Installation -TODO +Make sure to have installed [Elixir](https://elixir-lang.org/install.html) first. + +Run `mix deps.get`. ## Usage -TODO +Run `mix help lltest` for usage information. ## Copyright and License diff --git a/lib/jellygrinder.ex b/lib/jellygrinder.ex deleted file mode 100644 index 4da4ee2..0000000 --- a/lib/jellygrinder.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Jellygrinder.Grinder do - @moduledoc false -end diff --git a/lib/jellygrinder/application.ex b/lib/jellygrinder/application.ex new file mode 100644 index 0000000..efbd210 --- /dev/null +++ b/lib/jellygrinder/application.ex @@ -0,0 +1,16 @@ +defmodule Jellygrinder.Application do + @moduledoc false + + use Application + alias Jellygrinder.{ClientSupervisor, Coordinator} + + @impl true + def start(_mode, _opts) do + children = [ + ClientSupervisor, + Coordinator + ] + + Supervisor.start_link(children, strategy: :one_for_one) + end +end diff --git a/lib/jellygrinder/client_supervisor.ex b/lib/jellygrinder/client_supervisor.ex new file mode 100644 index 0000000..f625706 --- /dev/null +++ b/lib/jellygrinder/client_supervisor.ex @@ -0,0 +1,26 @@ +defmodule Jellygrinder.ClientSupervisor do + @moduledoc false + + use DynamicSupervisor + alias Jellygrinder.LLClient + + @spec start_link(term()) :: Supervisor.on_start() + def start_link(arg) do + DynamicSupervisor.start_link(__MODULE__, arg, name: __MODULE__) + end + + @spec spawn_client(module(), term()) :: DynamicSupervisor.on_start_child() + def spawn_client(client_module \\ LLClient, arg) do + DynamicSupervisor.start_child(__MODULE__, {client_module, arg}) + end + + @spec terminate() :: :ok + def terminate() do + DynamicSupervisor.stop(__MODULE__) + end + + @impl true + def init(_arg) do + DynamicSupervisor.init(strategy: :one_for_one) + end +end diff --git a/lib/jellygrinder/coordinator.ex b/lib/jellygrinder/coordinator.ex new file mode 100644 index 0000000..bf9a76a --- /dev/null +++ b/lib/jellygrinder/coordinator.ex @@ -0,0 +1,151 @@ +defmodule Jellygrinder.Coordinator do + @moduledoc false + + use GenServer, restart: :temporary + + require Logger + + alias Jellygrinder.ClientSupervisor + alias Jellygrinder.Coordinator.Config + + @spec run_test(Config.t()) :: :ok | no_return() + def run_test(config) do + ref = Process.monitor(__MODULE__) + GenServer.call(__MODULE__, {:run_test, config}) + + receive do + {:DOWN, ^ref, :process, _pid, reason} -> + if reason != :normal, + do: Logger.error("Coordinator process exited with reason #{inspect(reason)}") + + :ok + end + end + + @spec start_link(term()) :: GenServer.on_start() + def start_link(_args) do + GenServer.start_link(__MODULE__, nil, name: __MODULE__) + end + + @impl true + def init(_args) do + Logger.info("Coordinator: Init") + + {:ok, nil} + end + + @impl true + def handle_call({:run_test, config}, _from, _state) do + config = Config.fill_hls_url!(config) + + Logger.info(""" + Coordinator: Start of test + URL: #{config.url} + Clients: #{config.clients} + Time: #{config.time} s + Save results to: #{config.out_path} + """) + + Process.send_after(self(), :end_test, config.time * 1000) + send(self(), :spawn_client) + + state = %{ + uri: URI.parse(config.url), + clients: config.clients, + time: config.time, + spawn_interval: config.spawn_interval, + out_path: config.out_path, + client_count: 0, + results: [] + } + + {:reply, :ok, state} + end + + @impl true + def handle_cast({:result, r}, %{results: results} = state) do + r = amend_result(r, state) + + unless r.success do + Logger.warning( + "Coordinator: Request failed (from: #{r.process_name}, label: #{r.label}, code: #{r.response_code})" + ) + end + + {:noreply, %{state | results: [r | results]}} + end + + @impl true + def handle_info(:spawn_client, %{client_count: max_clients, clients: max_clients} = state) do + {:noreply, state} + end + + @impl true + def handle_info(:spawn_client, %{client_count: client_count} = state) do + Process.send_after(self(), :spawn_client, state.spawn_interval) + name = "client-#{client_count}" + + case ClientSupervisor.spawn_client(%{uri: state.uri, name: name}) do + {:ok, pid} -> + Logger.info("Coordinator: #{name} spawned at #{inspect(pid)}") + _ref = Process.monitor(pid) + + {:noreply, %{state | client_count: client_count + 1}} + + {:error, reason} -> + Logger.error("Coordinator: Error spawning #{name}: #{inspect(reason)}") + + {:noreply, state} + end + end + + @impl true + def handle_info(:end_test, %{results: results, out_path: out_path} = state) do + Logger.info("Coordinator: End of test") + + ClientSupervisor.terminate() + + Logger.info("Coordinator: Generating report...") + + results = + results + |> Enum.reverse() + |> Enum.map_join("", &serialize_result/1) + + Logger.info("Coordinator: Saving generated report to #{out_path}...") + File.write!(out_path, results_header() <> results) + Logger.info("Coordinator: Report saved successfully. Exiting") + + {:stop, :normal, state} + end + + @impl true + def handle_info({:DOWN, _ref, :process, pid, reason}, %{client_count: client_count} = state) do + Logger.warning("Coordinator: Child process #{inspect(pid)} died: #{inspect(reason)}") + + {:noreply, %{state | client_count: client_count - 1}} + end + + @impl true + def handle_info(msg, state) do + Logger.warning("Coordinator: Received unexpected message: #{inspect(msg)}") + + {:noreply, state} + end + + defp amend_result(result, %{client_count: client_count, uri: uri} = _state) do + request_url = uri |> Map.put(:path, result.path) |> URI.to_string() + + result + |> Map.put(:client_count, client_count) + |> Map.put(:url, request_url) + end + + defp results_header() do + "timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect\n" + end + + defp serialize_result(r) do + "#{r.timestamp},#{r.elapsed},#{r.label},#{r.response_code},,#{r.process_name},,#{r.success},#{r.failure_msg},#{r.bytes},-1,#{r.client_count},#{r.client_count},#{r.url},-1,-1,-1\n" + end +end diff --git a/lib/jellygrinder/coordinator/config.ex b/lib/jellygrinder/coordinator/config.ex new file mode 100644 index 0000000..25f7706 --- /dev/null +++ b/lib/jellygrinder/coordinator/config.ex @@ -0,0 +1,49 @@ +defmodule Jellygrinder.Coordinator.Config do + @moduledoc false + + @default_client_config [ + server_address: "localhost:5002", + server_api_token: "development", + secure?: false + ] + + @type t :: %__MODULE__{ + client_config: Jellyfish.Client.connection_options(), + url: String.t() | nil, + clients: pos_integer(), + time: pos_integer(), + spawn_interval: pos_integer(), + out_path: Path.t() + } + + defstruct client_config: @default_client_config, + url: nil, + clients: 500, + time: 300, + spawn_interval: 200, + out_path: "results.csv" + + @spec fill_hls_url!(t()) :: t() | no_return() + def fill_hls_url!(%{url: nil} = config) do + client_config = Keyword.merge(@default_client_config, config.client_config) + client = Jellyfish.Client.new(client_config) + + case Jellyfish.Room.get_all(client) do + {:ok, [room | _rest]} -> + protocol = if client_config[:secure?], do: "https", else: "http" + + %{ + config + | url: "#{protocol}://#{client_config[:server_address]}/hls/#{room.id}/index.m3u8" + } + + {:ok, []} -> + raise "No rooms present on Jellyfish" + + {:error, reason} -> + raise "Error communicating with Jellyfish: #{inspect(reason)}" + end + end + + def fill_hls_url!(config), do: config +end diff --git a/lib/jellygrinder/ll_client.ex b/lib/jellygrinder/ll_client.ex new file mode 100644 index 0000000..2b40449 --- /dev/null +++ b/lib/jellygrinder/ll_client.ex @@ -0,0 +1,179 @@ +defmodule Jellygrinder.LLClient do + @moduledoc false + + use GenServer, restart: :temporary + alias Jellygrinder.LLClient.ConnectionManager + + @max_partial_request_count 12 + @max_single_partial_request_retries 3 + + # in ms + @backoff 1000 + + @parent Jellygrinder.Coordinator + + @spec start_link(%{uri: URI.t(), name: String.t()}) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + @impl true + def init(opts) do + master_manifest_uri = opts.uri + {:ok, conn_manager} = ConnectionManager.start_link(master_manifest_uri) + + state = %{ + conn_manager: conn_manager, + name: opts.name, + base_path: Path.dirname(master_manifest_uri.path), + track_manifest_name: nil, + latest_partial: nil + } + + {:ok, state, {:continue, {:get_master_manifest, master_manifest_uri.path}}} + end + + @impl true + def handle_continue({:get_master_manifest, path}, state) do + case request(path, "master playlist", state) do + {:ok, master_manifest} -> + send(self(), :get_new_partials) + track_manifest_name = get_track_manifest_name(master_manifest) + + {:noreply, %{state | track_manifest_name: track_manifest_name}} + + {:error, _response} -> + {:stop, :missing_master_manifest, state} + end + end + + @impl true + def handle_info(:get_new_partials, state) do + path = Path.join(state.base_path, state.track_manifest_name) + query = create_track_manifest_query(state) + + case request(path <> query, "media playlist", state) do + {:ok, track_manifest} -> + latest_partial = + track_manifest + |> get_new_partials_info(state.latest_partial) + |> Stream.map(&request_partial(&1, state)) + |> Stream.take(-1) + |> Enum.to_list() + |> List.first(state.latest_partial) + + send(self(), :get_new_partials) + + {:noreply, %{state | latest_partial: latest_partial}} + + {:error, _response} -> + Process.send_after(self(), :get_new_partials, @backoff) + + {:noreply, state} + end + end + + @impl true + def handle_info(_msg, state) do + {:noreply, state} + end + + defp create_track_manifest_query(%{latest_partial: nil} = _state), do: "" + + defp create_track_manifest_query(%{latest_partial: latest_partial} = _state) do + [last_msn, last_part] = + Regex.run(~r/^muxed_segment_(\d+)_\w*_(\d+)_part.m4s$/, latest_partial.name, + capture: :all_but_first + ) + |> Enum.map(&String.to_integer/1) + + # This may not be the correct client behaviour, but it is handled by Jellyfish + # TODO: rewrite when the client starts handling preload hints + "?_HLS_msn=#{last_msn}&_HLS_part=#{last_part + 1}" + end + + defp request_partial(partial, state, retries \\ @max_single_partial_request_retries) + defp request_partial(partial, _state, 0), do: partial + + defp request_partial(partial, state, retries) do + path = Path.join(state.base_path, partial.name) + + case request(path, "media partial segment", state) do + {:ok, _content} -> partial + {:error, _reason} -> request_partial(partial, state, retries - 1) + end + end + + defp get_track_manifest_name(master_manifest) do + master_manifest + |> String.split("\n") + |> List.last() + end + + defp get_new_partials_info(track_manifest, latest_partial) do + track_manifest + |> trim_manifest(latest_partial) + |> then(&Regex.scan(~r/^#EXT-X-PART:DURATION=(.*),URI="(.*)"/m, &1, capture: :all_but_first)) + |> Enum.take(-@max_partial_request_count) + |> Enum.map(fn [duration, name] -> %{duration: String.to_float(duration), name: name} end) + end + + defp trim_manifest(manifest, nil) do + manifest + end + + # Trim the manifest, returning everything after `partial` + # If `partial` isn't present, return the entire manifest + defp trim_manifest(manifest, partial) do + manifest + |> String.split(partial.name, parts: 2) + |> Enum.at(1, manifest) + end + + defp request(path, label, state) do + timestamp = get_current_timestamp_ms() + start_time = System.monotonic_time() + maybe_response = ConnectionManager.get(state.conn_manager, path) + end_time = System.monotonic_time() + + request_info = %{ + timestamp: timestamp, + elapsed: System.convert_time_unit(end_time - start_time, :native, :millisecond), + label: label, + process_name: state.name, + path: path + } + + {result, data} = + case maybe_response do + {:ok, response} -> + success = response.status == 200 + data = Map.get(response, :data, "") + + {%{ + response_code: response.status, + success: success, + failure_msg: if(success, do: "", else: data), + bytes: byte_size(data) + }, data} + + {:error, reason} -> + {%{ + response_code: -1, + success: false, + failure_msg: inspect(reason), + bytes: -1 + }, ""} + end + + GenServer.cast(@parent, {:result, Map.merge(request_info, result)}) + + {if(result.success, do: :ok, else: :error), data} + end + + defp get_current_timestamp_ms() do + {megaseconds, seconds, microseconds} = :os.timestamp() + + megaseconds * 1_000_000_000 + seconds * 1000 + div(microseconds, 1000) + end +end diff --git a/lib/jellygrinder/ll_client/connection_manager.ex b/lib/jellygrinder/ll_client/connection_manager.ex new file mode 100644 index 0000000..b134d43 --- /dev/null +++ b/lib/jellygrinder/ll_client/connection_manager.ex @@ -0,0 +1,81 @@ +defmodule Jellygrinder.LLClient.ConnectionManager do + @moduledoc false + + use GenServer + + defstruct [:conn, requests: %{}] + + @connection_opts [protocols: [:http2]] + + @spec start_link(URI.t()) :: GenServer.on_start() + def start_link(uri) do + GenServer.start_link(__MODULE__, uri) + end + + @spec get(GenServer.server(), Path.t()) :: {:ok, map()} | {:error, term()} + def get(pid, path) do + GenServer.call(pid, {:get, path}) + end + + @impl true + def init(uri) do + case Mint.HTTP.connect(String.to_atom(uri.scheme), uri.host, uri.port, @connection_opts) do + {:ok, conn} -> + state = %__MODULE__{conn: conn} + {:ok, state} + + {:error, reason} -> + {:stop, reason} + end + end + + @impl true + def handle_call({:get, path}, from, state) do + case Mint.HTTP.request(state.conn, "GET", path, [], "") do + {:ok, conn, request_ref} -> + state = put_in(state.requests[request_ref], %{from: from, response: %{}}) + {:noreply, %{state | conn: conn}} + + {:error, conn, reason} -> + {:reply, {:error, reason}, %{state | conn: conn}} + end + end + + @impl true + def handle_info(message, state) do + case Mint.HTTP.stream(state.conn, message) do + :unknown -> + {:noreply, state} + + {:ok, conn, responses} -> + state = Enum.reduce(responses, state, &process_response/2) + {:noreply, %{state | conn: conn}} + end + end + + defp process_response({:status, request_ref, status}, state) do + put_in(state.requests[request_ref].response[:status], status) + end + + defp process_response({:headers, request_ref, headers}, state) do + put_in(state.requests[request_ref].response[:headers], headers) + end + + defp process_response({:data, request_ref, new_data}, state) do + update_in(state.requests[request_ref].response[:data], fn data -> (data || "") <> new_data end) + end + + defp process_response({:done, request_ref}, state) do + {%{response: response, from: from}, state} = pop_in(state.requests[request_ref]) + GenServer.reply(from, {:ok, response}) + + state + end + + defp process_response({:error, request_ref, reason}, state) do + {%{from: from}, state} = pop_in(state.requests[request_ref]) + GenServer.reply(from, {:error, reason}) + + state + end +end diff --git a/lib/mix/tasks/lltest.ex b/lib/mix/tasks/lltest.ex new file mode 100644 index 0000000..d81f9d2 --- /dev/null +++ b/lib/mix/tasks/lltest.ex @@ -0,0 +1,86 @@ +defmodule Mix.Tasks.Lltest do + @shortdoc "Run LL-HLS stress test" + @moduledoc """ + # Name + + `mix lltest` - #{@shortdoc} + + # Synopsis + + ``` + mix lltest [--url ] [--clients ] [--time ] + [--spawn-interval ] [--out-path ] + [--jellyfish-address
] [--jellyfish-token ] [--secure] + ``` + + # Description + + Mix task for running stress-tests on a Jellyfish serving LL-HLS. + + This tool primarily tests the load handling capability and performance + of the media server. The test simulates multiple clients requesting LL-HLS + content streams concurrently over a specified duration. + + It saves a CSV file with test results after the full duration of the test has passed, + which means that if the test is interrupted e.g. using Ctrl-C, the results will NOT be saved. + + # Available options + + * `--url ` - URL of the master HLS manifest. This can be inferred from Jellyfish, see below + * `--clients ` - Number of client connections to simulate. Defaults to 500 + * `--time ` - Duration of the test. Defaults to 300 seconds + * `--spawn-interval ` - Interval at which to spawn new clients. Defaults to 200 milliseconds + * `--out-path ` - Path to store the CSV with test results. Defaults to "results.csv" + + If `--url ` is not passed, the tool will attempt to infer the URL by communicating with Jellyfish. + This uses the following options: + + * `--jellyfish-address
` - Address (structured `:`) of Jellyfish. Defaults to "localhost:5002" + * `--jellyfish-token ` - Jellyfish token. Defaults to "development" + * `--secure` - By default, the tool will try to communicate with Jellyfish using HTTP. + If this option is passed, it will use HTTPS instead + + # Example command + + `mix lltest --jellyfish-address my-jellyfish.org:443 --jellyfish-token my-token --secure --clients 2000 --time 600` + """ + + use Mix.Task + alias Jellygrinder.Coordinator + + @impl true + def run(argv) do + Application.ensure_all_started(:jellygrinder) + + {opts, _argv, _errors} = + OptionParser.parse(argv, + strict: [ + jellyfish_address: :string, + jellyfish_token: :string, + secure: :boolean, + url: :string, + clients: :integer, + time: :integer, + spawn_interval: :integer, + out_path: :string + ] + ) + + client_config = + Enum.reduce(opts, Keyword.new(), fn {key, value}, config -> + case key do + :jellyfish_address -> Keyword.put(config, :server_address, value) + :jellyfish_token -> Keyword.put(config, :server_api_token, value) + :secure -> Keyword.put(config, :secure?, value) + _other -> config + end + end) + + coordinator_config = + opts + |> Keyword.put(:client_config, client_config) + |> then(&struct(Coordinator.Config, &1)) + + Coordinator.run_test(coordinator_config) + end +end diff --git a/mix.exs b/mix.exs index 2885d73..db6bf7c 100644 --- a/mix.exs +++ b/mix.exs @@ -15,7 +15,8 @@ defmodule Jellygrinder.MixProject do def application do [ - extra_applications: [:logger] + extra_applications: [:logger], + mod: {Jellygrinder.Application, []} ] end @@ -23,6 +24,9 @@ defmodule Jellygrinder.MixProject do defp deps do [ + {:mint, "~> 1.5"}, + {:castore, "~> 1.0"}, + {:jellyfish_server_sdk, github: "jellyfish-dev/elixir_server_sdk"}, {:dialyxir, ">= 0.0.0", only: :dev, runtime: false}, {:credo, ">= 0.0.0", only: :dev, runtime: false} ] @@ -30,7 +34,8 @@ defmodule Jellygrinder.MixProject do defp dialyzer() do opts = [ - flags: [:error_handling] + flags: [:error_handling], + plt_add_apps: [:mix] ] if System.get_env("CI") == "true" do diff --git a/mix.lock b/mix.lock index 7168e3c..56583d9 100644 --- a/mix.lock +++ b/mix.lock @@ -1,8 +1,17 @@ %{ "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, - "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, + "castore": {:hex, :castore, "1.0.3", "7130ba6d24c8424014194676d608cb989f62ef8039efd50ff4b3f33286d06db8", [:mix], [], "hexpm", "680ab01ef5d15b161ed6a95449fac5c6b8f60055677a8e79acf01b27baa4390b"}, + "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"}, "dialyxir": {:hex, :dialyxir, "1.4.1", "a22ed1e7bd3a3e3f197b68d806ef66acb61ee8f57b3ac85fc5d57354c5482a93", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "84b795d6d7796297cca5a3118444b80c7d94f7ce247d49886e7c291e1ae49801"}, + "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, + "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, + "jellyfish_server_sdk": {:git, "https://github.com/jellyfish-dev/elixir_server_sdk.git", "c997733cd6fc59a4cd821e1cdc6362e04e6e85f9", []}, + "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, + "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, + "protobuf": {:hex, :protobuf, "0.12.0", "58c0dfea5f929b96b5aa54ec02b7130688f09d2de5ddc521d696eec2a015b223", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "75fa6cbf262062073dd51be44dd0ab940500e18386a6c4e87d5819a58964dc45"}, + "tesla": {:hex, :tesla, "1.7.0", "a62dda2f80d4f8a925eb7b8c5b78c461e0eb996672719fe1a63b26321a5f8b4e", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "2e64f01ebfdb026209b47bc651a0e65203fcff4ae79c11efb73c4852b00dc313"}, + "websockex": {:hex, :websockex, "0.4.3", "92b7905769c79c6480c02daacaca2ddd49de936d912976a4d3c923723b647bf0", [:mix], [], "hexpm", "95f2e7072b85a3a4cc385602d42115b73ce0b74a9121d0d6dbbf557645ac53e4"}, }