Merge pull request blockscout#1543 from poanetwork/gs-block-internal-…

…transactions

Use trace_replayBlockTransactions API for faster tracing

.dialyzer-ignore

@@ -1,6 +1,6 @@
 :0: Unknown function 'Elixir.ExUnit.Callbacks':'__merge__'/3
 :0: Unknown function 'Elixir.ExUnit.CaseTemplate':'__proxy__'/2
 :0: Unknown type 'Elixir.Map':t/0
-apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex:390: Function timestamp_to_datetime/1 has no local return
+apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex:400: Function timestamp_to_datetime/1 has no local return
 apps/explorer/lib/explorer/repo/prometheus_logger.ex:8: Function microseconds_time/1 has no local return
 apps/explorer/lib/explorer/repo/prometheus_logger.ex:8: The call 'Elixir.System':convert_time_unit(__@1::any(),'native','microseconds') breaks the contract (integer(),time_unit() | 'native',time_unit() | 'native') -> integer()

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex

@@ -264,6 +264,16 @@ defmodule EthereumJSONRPC do
     )
   end
 
+  @doc """
+  Fetches internal transactions for entire blocks from variant API.
+  """
+  def fetch_block_internal_transactions(params_list, json_rpc_named_arguments) when is_list(params_list) do
+    Keyword.fetch!(json_rpc_named_arguments, :variant).fetch_block_internal_transactions(
+      params_list,
+      json_rpc_named_arguments
+    )
+  end
+
   @doc """
   Fetches pending transactions from variant API.
   """

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/ganache.ex

@@ -21,6 +21,14 @@ defmodule EthereumJSONRPC.Ganache do
   @impl EthereumJSONRPC.Variant
   def fetch_internal_transactions(_transactions_params, _json_rpc_named_arguments), do: :ignore
 
+  @doc """
+  Internal transaction fetching is not currently supported for Ganache.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_block_internal_transactions(_block_range, _json_rpc_named_arguments), do: :ignore
+
   @doc """
   Pending transaction fetching is not supported currently for Ganache.
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -32,6 +32,14 @@ defmodule EthereumJSONRPC.Geth do
     end
   end
 
+  @doc """
+  Internal transaction fetching for entire blocks is not currently supported for Geth.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_block_internal_transactions(_block_range, _json_rpc_named_arguments), do: :ignore
+
   @doc """
   Pending transaction fetching is not supported currently for Geth.
 

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/parity.ex

@@ -26,18 +26,26 @@ defmodule EthereumJSONRPC.Parity do
     end
   end
 
+  @doc """
+  Internal transaction fetching for individual transactions is no longer supported for Parity.
+
+  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  """
+  @impl EthereumJSONRPC.Variant
+  def fetch_internal_transactions(_transactions_params, _json_rpc_named_arguments), do: :ignore
+
   @doc """
   Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Parity trace URL.
   """
   @impl EthereumJSONRPC.Variant
-  def fetch_internal_transactions(transactions_params, json_rpc_named_arguments) when is_list(transactions_params) do
-    id_to_params = id_to_params(transactions_params)
+  def fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments) when is_list(block_numbers) do
+    id_to_params = id_to_params(block_numbers)
 
     with {:ok, responses} <-
            id_to_params
-           |> trace_replay_transaction_requests()
+           |> trace_replay_block_transactions_requests()
            |> json_rpc(json_rpc_named_arguments) do
-      trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+      trace_replay_block_transactions_responses_to_internal_transactions_params(responses, id_to_params)
     end
   end
 
@@ -68,9 +76,9 @@ defmodule EthereumJSONRPC.Parity do
     Enum.map(block_numbers, &%{block_quantity: integer_to_quantity(&1)})
   end
 
-  defp trace_replay_transaction_responses_to_internal_transactions_params(responses, id_to_params)
+  defp trace_replay_block_transactions_responses_to_internal_transactions_params(responses, id_to_params)
        when is_list(responses) and is_map(id_to_params) do
-    with {:ok, traces} <- trace_replay_transaction_responses_to_traces(responses, id_to_params) do
+    with {:ok, traces} <- trace_replay_block_transactions_responses_to_traces(responses, id_to_params) do
       params =
         traces
         |> Traces.to_elixir()
@@ -80,10 +88,10 @@ defmodule EthereumJSONRPC.Parity do
     end
   end
 
-  defp trace_replay_transaction_responses_to_traces(responses, id_to_params)
+  defp trace_replay_block_transactions_responses_to_traces(responses, id_to_params)
        when is_list(responses) and is_map(id_to_params) do
     responses
-    |> Enum.map(&trace_replay_transaction_response_to_traces(&1, id_to_params))
+    |> Enum.map(&trace_replay_block_transactions_response_to_traces(&1, id_to_params))
     |> Enum.reduce(
       {:ok, []},
       fn
@@ -115,48 +123,48 @@ defmodule EthereumJSONRPC.Parity do
     end
   end
 
-  defp trace_replay_transaction_response_to_traces(%{id: id, result: %{"trace" => traces}}, id_to_params)
-       when is_list(traces) and is_map(id_to_params) do
-    %{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =
-      Map.fetch!(id_to_params, id)
+  defp trace_replay_block_transactions_response_to_traces(%{id: id, result: results}, id_to_params)
+       when is_list(results) and is_map(id_to_params) do
+    block_number = Map.fetch!(id_to_params, id)
 
     annotated_traces =
-      traces
+      results
       |> Stream.with_index()
-      |> Enum.map(fn {trace, index} ->
-        Map.merge(trace, %{
-          "blockNumber" => block_number,
-          "index" => index,
-          "transactionIndex" => transaction_index,
-          "transactionHash" => transaction_hash
-        })
+      |> Enum.flat_map(fn {%{"trace" => traces, "transactionHash" => transaction_hash}, transaction_index} ->
+        traces
+        |> Stream.with_index()
+        |> Enum.map(fn {trace, index} ->
+          Map.merge(trace, %{
+            "blockNumber" => block_number,
+            "transactionHash" => transaction_hash,
+            "transactionIndex" => transaction_index,
+            "index" => index
+          })
+        end)
       end)
 
     {:ok, annotated_traces}
   end
 
-  defp trace_replay_transaction_response_to_traces(%{id: id, error: error}, id_to_params)
+  defp trace_replay_block_transactions_response_to_traces(%{id: id, error: error}, id_to_params)
        when is_map(id_to_params) do
-    %{block_number: block_number, hash_data: transaction_hash, transaction_index: transaction_index} =
-      Map.fetch!(id_to_params, id)
+    block_number = Map.fetch!(id_to_params, id)
 
     annotated_error =
       Map.put(error, :data, %{
-        "blockNumber" => block_number,
-        "transactionIndex" => transaction_index,
-        "transactionHash" => transaction_hash
+        "blockNumber" => block_number
       })
 
     {:error, annotated_error}
   end
 
-  defp trace_replay_transaction_requests(id_to_params) when is_map(id_to_params) do
-    Enum.map(id_to_params, fn {id, %{hash_data: hash_data}} ->
-      trace_replay_transaction_request(%{id: id, hash_data: hash_data})
+  defp trace_replay_block_transactions_requests(id_to_params) when is_map(id_to_params) do
+    Enum.map(id_to_params, fn {id, block_number} ->
+      trace_replay_block_transactions_request(%{id: id, block_number: block_number})
     end)
   end
 
-  defp trace_replay_transaction_request(%{id: id, hash_data: hash_data}) do
-    request(%{id: id, method: "trace_replayTransaction", params: [hash_data, ["trace"]]})
+  defp trace_replay_block_transactions_request(%{id: id, block_number: block_number}) do
+    request(%{id: id, method: "trace_replayBlockTransactions", params: [integer_to_quantity(block_number), ["trace"]]})
   end
 end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/variant.ex

@@ -43,6 +43,22 @@ defmodule EthereumJSONRPC.Variant do
               EthereumJSONRPC.json_rpc_named_arguments()
             ) :: {:ok, [internal_transaction_params]} | {:error, reason :: term} | :ignore
 
+  @doc """
+  Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the variant of the Ethereum JSONRPC API.
+  Uses API for fetching all internal transactions in the block
+
+  ## Returns
+
+   * `{:ok, [internal_transaction_params]}` - internal transactions were successfully fetched for all blocks
+   * `{:error, reason}` - there was one or more errors with `reason` in fetching at least one of the blocks'
+       internal transactions
+   * `:ignore` - the variant does not support fetching internal transactions.
+  """
+  @callback fetch_block_internal_transactions(
+              [EthereumJSONRPC.block_number()],
+              EthereumJSONRPC.json_rpc_named_arguments()
+            ) :: {:ok, [internal_transaction_params]} | {:error, reason :: term} | :ignore
+
   @doc """
   Fetch the `t:Explorer.Chain.Transaction.changeset/2` params for pending transactions from the variant of the Ethereum
   JSONRPC API.

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/geth_test.exs

@@ -77,6 +77,12 @@ defmodule EthereumJSONRPC.GethTest do
     end
   end
 
+  describe "fetch_block_internal_transactions/1" do
+    test "is not supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
+      EthereumJSONRPC.Geth.fetch_block_internal_transactions([], json_rpc_named_arguments)
+    end
+  end
+
   describe "fetch_pending_transactions/1" do
     test "is not supported", %{json_rpc_named_arguments: json_rpc_named_arguments} do
       EthereumJSONRPC.Geth.fetch_pending_transactions(json_rpc_named_arguments)

apps/ethereum_jsonrpc/test/ethereum_jsonrpc/http/mox_test.exs

@@ -103,44 +103,38 @@ defmodule EthereumJSONRPC.HTTP.MoxTest do
     test "transparently splits batch payloads that would trigger a 504 Gateway Timeout", %{
       json_rpc_named_arguments: json_rpc_named_arguments
     } do
-      transaction_hashes = ~w(0x196c2579f30077e8df0994e185d724331350c2bdb0f5d4e48b9e83f1e149cc28
-         0x19eb948514a971bcd3ab737083bbdb32da233fff2ba70490bb0a36937a418006
-         0x1a1039899fd07a5fd81faf2ec11ca24fc6023d486d4156095688a29b3bf06b7b
-         0x1a942061ed6cf0736b194732bb6e1edfcbc50cc004e0cdad79335b3e40e23c9c
-         0x1bdec995deaa0e5b53cc7a0b84eaff39da90f5e507fdb4360881ff31f824d918
-         0x1c26758e003b0bc89ac7e3e6e87c6fc76dfb8d878dc530055e6a34f4d557cb1c
-         0x1d592be82979bd1cc320eb70d4bb1d61226d78baa9e57e2a12b24345f81ce3bd
-         0x1e57e7ce2941c6108e899f786fe339fa50ab053e47fbdcbf5979f475042c6dd8
-         0x1ec1f9c31a0f43f7e684bfa20e422d7d8a343f81c517be1e30f149618ae306f2
-         0x221aaf59f7a05702f0f53744b4fdb5f74e3c6fdade7324fda342cc1ebc73e01c)
+      block_numbers = [862_272, 862_273, 862_274, 862_275, 862_276, 862_277, 862_278, 862_279, 862_280, 862_281]
 
       if json_rpc_named_arguments[:transport_options][:http] == EthereumJSONRPC.HTTP.Mox do
         EthereumJSONRPC.HTTP.Mox
         |> expect(:json_rpc, fn _url, _json, _options ->
-          {:ok, %{body: "504 Gateway Timeout", status_code: 413}}
+          {:ok, %{body: "504 Gateway Timeout", status_code: 504}}
         end)
         |> expect(:json_rpc, fn _url, json, _options ->
           json_binary = IO.iodata_to_binary(json)
 
-          refute json_binary =~ "0x221aaf59f7a05702f0f53744b4fdb5f74e3c6fdade7324fda342cc1ebc73e01c"
-          assert json_binary =~ "0x1bdec995deaa0e5b53cc7a0b84eaff39da90f5e507fdb4360881ff31f824d918"
+          refute json_binary =~ "0xD2849"
+          assert json_binary =~ "0xD2844"
 
           body =
             0..4
             |> Enum.map(fn id ->
               %{
                 jsonrpc: "2.0",
                 id: id,
-                result: %{
-                  "trace" => [
-                    %{
-                      "type" => "create",
-                      "action" => %{"from" => "0x", "gas" => "0x0", "init" => "0x", "value" => "0x0"},
-                      "traceAddress" => "0x",
-                      "result" => %{"address" => "0x", "code" => "0x", "gasUsed" => "0x0"}
-                    }
-                  ]
-                }
+                result: [
+                  %{
+                    "trace" => [
+                      %{
+                        "type" => "create",
+                        "action" => %{"from" => "0x", "gas" => "0x0", "init" => "0x", "value" => "0x0"},
+                        "traceAddress" => "0x",
+                        "result" => %{"address" => "0x", "code" => "0x", "gasUsed" => "0x0"}
+                      }
+                    ],
+                    "transactionHash" => "0x221aaf59f7a05702f0f53744b4fdb5f74e3c6fdade7324fda342cc1ebc73e01c"
+                  }
+                ]
               }
             end)
             |> Jason.encode!()
@@ -150,26 +144,29 @@ defmodule EthereumJSONRPC.HTTP.MoxTest do
         |> expect(:json_rpc, fn _url, json, _options ->
           json_binary = IO.iodata_to_binary(json)
 
-          refute json_binary =~ "0x1bdec995deaa0e5b53cc7a0b84eaff39da90f5e507fdb4360881ff31f824d918"
-          assert json_binary =~ "0x1c26758e003b0bc89ac7e3e6e87c6fc76dfb8d878dc530055e6a34f4d557cb1c"
-          assert json_binary =~ "0x221aaf59f7a05702f0f53744b4fdb5f74e3c6fdade7324fda342cc1ebc73e01c"
+          refute json_binary =~ "0xD2844"
+          assert json_binary =~ "0xD2845"
+          assert json_binary =~ "0xD2849"
 
           body =
             5..9
             |> Enum.map(fn id ->
               %{
                 jsonrpc: "2.0",
                 id: id,
-                result: %{
-                  "trace" => [
-                    %{
-                      "type" => "create",
-                      "action" => %{"from" => "0x", "gas" => "0x0", "init" => "0x", "value" => "0x0"},
-                      "traceAddress" => "0x",
-                      "result" => %{"address" => "0x", "code" => "0x", "gasUsed" => "0x0"}
-                    }
-                  ]
-                }
+                result: [
+                  %{
+                    "trace" => [
+                      %{
+                        "type" => "create",
+                        "action" => %{"from" => "0x", "gas" => "0x0", "init" => "0x", "value" => "0x0"},
+                        "traceAddress" => "0x",
+                        "result" => %{"address" => "0x", "code" => "0x", "gasUsed" => "0x0"}
+                      }
+                    ],
+                    "transactionHash" => "0x221aaf59f7a05702f0f53744b4fdb5f74e3c6fdade7324fda342cc1ebc73e01c"
+                  }
+                ]
               }
             end)
             |> Jason.encode!()
@@ -178,24 +175,19 @@ defmodule EthereumJSONRPC.HTTP.MoxTest do
         end)
       end
 
-      transactions_params =
-        Enum.map(transaction_hashes, fn hash_data ->
-          %{block_number: 0, hash_data: hash_data, gas: 1_000_000, transaction_index: 0}
-        end)
-
       assert {:ok, responses} =
-               EthereumJSONRPC.fetch_internal_transactions(transactions_params, json_rpc_named_arguments)
+               EthereumJSONRPC.fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments)
 
-      assert Enum.count(responses) == Enum.count(transactions_params)
+      assert Enum.count(responses) == Enum.count(block_numbers)
 
-      transaction_hash_set = MapSet.new(transaction_hashes)
+      block_number_set = MapSet.new(block_numbers)
 
-      response_transaction_hash_set =
-        Enum.into(responses, MapSet.new(), fn %{transaction_hash: transaction_hash} ->
-          transaction_hash
+      response_block_number_set =
+        Enum.into(responses, MapSet.new(), fn %{block_number: block_number} ->
+          block_number
         end)
 
-      assert MapSet.equal?(response_transaction_hash_set, transaction_hash_set)
+      assert MapSet.equal?(response_block_number_set, block_number_set)
     end
   end