feat: MUD API support (#9869)

* feat: mud support

* chore: fix ci warnings

* feat: skip missing schemas

* ci: build redstone image

* chore: fix dialyzer

* chore: remove noop migration

* feat: full-text table search

* fix: don't show deleted records

* fix: type specs and dializer fixes

* feat: checksum addresses

* fix: handle invalid params

* chore: add missing envs

.github/workflows/publish-docker-image-for-redstone.yml

@@ -0,0 +1,44 @@
+name: Redstone Publish Docker image
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - production-redstone
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: ${{ vars.RELEASE_VERSION }}
+      DOCKER_CHAIN_NAME: redstone
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repo
+        uses: ./.github/actions/setup-repo-and-short-sha
+        with:
+          docker-username: ${{ secrets.DOCKER_USERNAME }}
+          docker-password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:latest, blockscout/blockscout-${{ env.DOCKER_CHAIN_NAME }}:${{ env.RELEASE_VERSION }}-postrelease-${{ env.SHORT_SHA }}
+          platforms: |
+            linux/amd64
+            linux/arm64/v8
+          build-args: |
+            CACHE_EXCHANGE_RATES_PERIOD=
+            API_V1_READ_METHODS_DISABLED=false
+            DISABLE_WEBAPP=false
+            API_V1_WRITE_METHODS_DISABLED=false
+            CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
+            ADMIN_PANEL_ENABLED=false
+            CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
+            BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta.+commit.${{ env.SHORT_SHA }}
+            RELEASE_VERSION=${{ env.RELEASE_VERSION }}
+            CHAIN_TYPE=optimism
+            MUD_INDEXER_ENABLED=true

.github/workflows/release-redstone.yml

@@ -0,0 +1,46 @@
+name: Release for Redstone
+
+on:
+  release:
+    types: [published]
+
+env:
+  OTP_VERSION: ${{ vars.OTP_VERSION }}
+  ELIXIR_VERSION: ${{ vars.ELIXIR_VERSION }}
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    env:
+      RELEASE_VERSION: ${{ vars.RELEASE_VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repo
+        uses: ./.github/actions/setup-repo
+        with:
+          docker-username: ${{ secrets.DOCKER_USERNAME }}
+          docker-password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push Docker image for Redstone
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: blockscout/blockscout-redstone:latest, blockscout/blockscout-redstone:${{ env.RELEASE_VERSION }}
+          platforms: |
+            linux/amd64
+            linux/arm64/v8
+          build-args: |
+            CACHE_EXCHANGE_RATES_PERIOD=
+            API_V1_READ_METHODS_DISABLED=false
+            DISABLE_WEBAPP=false
+            API_V1_WRITE_METHODS_DISABLED=false
+            CACHE_TOTAL_GAS_USAGE_COUNTER_ENABLED=
+            ADMIN_PANEL_ENABLED=false
+            CACHE_ADDRESS_WITH_BALANCES_UPDATE_INTERVAL=
+            BLOCKSCOUT_VERSION=v${{ env.RELEASE_VERSION }}-beta
+            RELEASE_VERSION=${{ env.RELEASE_VERSION }}
+            CHAIN_TYPE=optimism
+            MUD_INDEXER_ENABLED=true

apps/block_scout_web/lib/block_scout_web/api_router.ex

@@ -346,6 +346,18 @@ defmodule BlockScoutWeb.ApiRouter do
         get("/batches/:batch_number", V2.ZkSyncController, :batch)
       end
     end
+
+    scope "/mud" do
+      if Application.compile_env(:explorer, Explorer.Chain.Mud)[:enabled] do
+        get("/worlds", V2.MudController, :worlds)
+        get("/worlds/count", V2.MudController, :worlds_count)
+        get("/worlds/:world/tables", V2.MudController, :world_tables)
+        get("/worlds/:world/tables/count", V2.MudController, :world_tables_count)
+        get("/worlds/:world/tables/:table_id/records", V2.MudController, :world_table_records)
+        get("/worlds/:world/tables/:table_id/records/count", V2.MudController, :world_table_records_count)
+        get("/worlds/:world/tables/:table_id/records/:record_id", V2.MudController, :world_table_record)
+      end
+    end
   end
 
   scope "/v1/graphql" do

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/mud_controller.ex

@@ -0,0 +1,261 @@
+defmodule BlockScoutWeb.API.V2.MudController do
+  use BlockScoutWeb, :controller
+
+  import BlockScoutWeb.Chain,
+    only: [
+      next_page_params: 4,
+      split_list_by_page: 1,
+      default_paging_options: 0
+    ]
+
+  import BlockScoutWeb.PagingHelper, only: [mud_records_sorting: 1]
+
+  alias Explorer.Chain.{Data, Hash, Mud, Mud.Schema.FieldSchema, Mud.Table}
+
+  action_fallback(BlockScoutWeb.API.V2.FallbackController)
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds` endpoint.
+  """
+  @spec worlds(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def worlds(conn, params) do
+    {worlds, next_page} =
+      params
+      |> mud_paging_options(["world"], [Hash.Address])
+      |> Mud.worlds_list()
+      |> split_list_by_page()
+
+    next_page_params =
+      next_page_params(next_page, worlds, conn.query_params, fn item ->
+        %{"world" => item}
+      end)
+
+    conn
+    |> put_status(200)
+    |> render(:worlds, %{worlds: worlds, next_page_params: next_page_params})
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/count` endpoint.
+  """
+  @spec worlds_count(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def worlds_count(conn, _params) do
+    count = Mud.worlds_count()
+
+    conn
+    |> put_status(200)
+    |> render(:count, %{count: count})
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/:world/tables` endpoint.
+  """
+  @spec world_tables(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def world_tables(conn, %{"world" => world_param} = params) do
+    with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do
+      options = params |> mud_paging_options(["table_id"], [Hash.Full]) |> Keyword.merge(mud_tables_filter(params))
+
+      {tables, next_page} =
+        world
+        |> Mud.world_tables(options)
+        |> split_list_by_page()
+
+      next_page_params =
+        next_page_params(next_page, tables, conn.query_params, fn item ->
+          %{"table_id" => item |> elem(0)}
+        end)
+
+      conn
+      |> put_status(200)
+      |> render(:tables, %{tables: tables, next_page_params: next_page_params})
+    end
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/count` endpoint.
+  """
+  @spec world_tables_count(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def world_tables_count(conn, %{"world" => world_param} = params) do
+    with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)} do
+      options = params |> mud_tables_filter()
+
+      count = Mud.world_tables_count(world, options)
+
+      conn
+      |> put_status(200)
+      |> render(:count, %{count: count})
+    end
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records` endpoint.
+  """
+  @spec world_table_records(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def world_table_records(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do
+    with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)},
+         {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)},
+         {:ok, schema} <- Mud.world_table_schema(world, table_id) do
+      options =
+        params
+        |> mud_paging_options(["key_bytes", "key0", "key1"], [Data, Hash.Full, Hash.Full])
+        |> Keyword.merge(mud_records_filter(params, schema))
+        |> Keyword.merge(mud_records_sorting(params))
+
+      {records, next_page} = world |> Mud.world_table_records(table_id, options) |> split_list_by_page()
+
+      blocks = Mud.preload_records_timestamps(records)
+
+      next_page_params =
+        next_page_params(next_page, records, conn.query_params, fn item ->
+          keys = [item.key_bytes, item.key0, item.key1] |> Enum.filter(&(!is_nil(&1)))
+          ["key_bytes", "key0", "key1"] |> Enum.zip(keys) |> Enum.into(%{})
+        end)
+
+      conn
+      |> put_status(200)
+      |> render(:records, %{
+        records: records,
+        table_id: table_id,
+        schema: schema,
+        blocks: blocks,
+        next_page_params: next_page_params
+      })
+    end
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/count` endpoint.
+  """
+  @spec world_table_records_count(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def world_table_records_count(conn, %{"world" => world_param, "table_id" => table_id_param} = params) do
+    with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)},
+         {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)},
+         {:ok, schema} <- Mud.world_table_schema(world, table_id) do
+      options = params |> mud_records_filter(schema)
+
+      count = Mud.world_table_records_count(world, table_id, options)
+
+      conn
+      |> put_status(200)
+      |> render(:count, %{count: count})
+    end
+  end
+
+  @doc """
+    Function to handle GET requests to `/api/v2/mud/worlds/:world/tables/:table_id/records/:record_id` endpoint.
+  """
+  @spec world_table_record(Plug.Conn.t(), map()) :: Plug.Conn.t()
+  def world_table_record(
+        conn,
+        %{"world" => world_param, "table_id" => table_id_param, "record_id" => record_id_param} = _params
+      ) do
+    with {:format, {:ok, world}} <- {:format, Hash.Address.cast(world_param)},
+         {:format, {:ok, table_id}} <- {:format, Hash.Full.cast(table_id_param)},
+         {:format, {:ok, record_id}} <- {:format, Data.cast(record_id_param)},
+         {:ok, schema} <- Mud.world_table_schema(world, table_id),
+         {:ok, record} <- Mud.world_table_record(world, table_id, record_id) do
+      blocks = Mud.preload_records_timestamps([record])
+
+      conn
+      |> put_status(200)
+      |> render(:record, %{record: record, table_id: table_id, schema: schema, blocks: blocks})
+    end
+  end
+
+  defp mud_tables_filter(params) do
+    Enum.reduce(params, [], fn {key, value}, acc ->
+      case key do
+        "filter_namespace" ->
+          Keyword.put(acc, :filter_namespace, parse_namespace_string(value))
+
+        "q" ->
+          Keyword.put(acc, :filter_search, parse_search_string(value))
+
+        _ ->
+          acc
+      end
+    end)
+  end
+
+  defp parse_namespace_string(namespace) do
+    filter =
+      case namespace do
+        nil -> {:ok, nil}
+        "0x" <> hex -> Base.decode16(hex, case: :mixed)
+        str -> {:ok, str}
+      end
+
+    case filter do
+      {:ok, ns} when is_binary(ns) and byte_size(ns) <= 14 ->
+        ns |> String.pad_trailing(14, <<0>>)
+
+      _ ->
+        nil
+    end
+  end
+
+  defp parse_search_string(q) do
+    # If the search string looks like hex-encoded table id or table full name,
+    # we try to parse and filter by that table id directly.
+    # Otherwise we do a full-text search of given string inside table id.
+    with :error <- Hash.Full.cast(q),
+         :error <- Table.table_full_name_to_table_id(q) do
+      q
+    else
+      {:ok, table_id} -> table_id
+    end
+  end
+
+  defp mud_records_filter(params, schema) do
+    Enum.reduce(params, [], fn {key, value}, acc ->
+      case key do
+        "filter_key0" -> Keyword.put(acc, :filter_key0, encode_filter(value, schema, 0))
+        "filter_key1" -> Keyword.put(acc, :filter_key1, encode_filter(value, schema, 1))
+        _ -> acc
+      end
+    end)
+  end
+
+  defp encode_filter(value, schema, field_idx) do
+    case value do
+      "false" ->
+        <<0::256>>
+
+      "true" ->
+        <<1::256>>
+
+      "0x" <> hex ->
+        bin = Base.decode16!(hex, case: :mixed)
+        # addresses are padded to 32 bytes with zeros on the right
+        if FieldSchema.type_of(schema.key_schema, field_idx) == 97 do
+          <<0::size(256 - byte_size(bin) * 8), bin::binary>>
+        else
+          <<bin::binary, 0::size(256 - byte_size(bin) * 8)>>
+        end
+
+      dec ->
+        num = dec |> Integer.parse() |> elem(0)
+        <<num::256>>
+    end
+  end
+
+  defp mud_paging_options(params, keys, types) do
+    page_key =
+      keys
+      |> Enum.zip(types)
+      |> Enum.reduce(%{}, fn {key, type}, acc ->
+        with param when param != nil <- Map.get(params, key),
+             {:ok, val} <- type.cast(param) do
+          acc |> Map.put(String.to_existing_atom(key), val)
+        else
+          _ -> acc
+        end
+      end)
+
+    if page_key == %{} do
+      [paging_options: default_paging_options()]
+    else
+      [paging_options: %{default_paging_options() | key: page_key}]
+    end
+  end
+end

apps/block_scout_web/lib/block_scout_web/paging_helper.ex

@@ -303,4 +303,21 @@ defmodule BlockScoutWeb.PagingHelper do
     do: [{:dynamic, :blocks_validated, :desc_nulls_last, ValidatorStability.dynamic_validated_blocks()}]
 
   defp do_validators_stability_sorting(_, _), do: []
+
+  @spec mud_records_sorting(%{required(String.t()) => String.t()}) :: [
+          {:sorting, SortingHelper.sorting_params()}
+        ]
+  def mud_records_sorting(%{"sort" => sort_field, "order" => order}) do
+    [sorting: do_mud_records_sorting(sort_field, order)]
+  end
+
+  def mud_records_sorting(_), do: []
+
+  defp do_mud_records_sorting("key_bytes", "asc"), do: [asc_nulls_first: :key_bytes]
+  defp do_mud_records_sorting("key_bytes", "desc"), do: [desc_nulls_last: :key_bytes]
+  defp do_mud_records_sorting("key0", "asc"), do: [asc_nulls_first: :key0]
+  defp do_mud_records_sorting("key0", "desc"), do: [desc_nulls_last: :key0]
+  defp do_mud_records_sorting("key1", "asc"), do: [asc_nulls_first: :key1]
+  defp do_mud_records_sorting("key1", "desc"), do: [desc_nulls_last: :key1]
+  defp do_mud_records_sorting(_, _), do: []
 end