Add protobuf code comments as Elixir module documentation (#352)

* Add protobuf code comments as moduledoc

* remove debug docs

* Remove grpc dep

* Refactor test to avoid dependency on grpc library

* Ensure moduledoc is set even if modules don't have comments

* Fix warning during code generation with multi-line comments

---------

Co-authored-by: v0idpwn <v0idpwn@gmail.com>

lib/protobuf/protoc/context.ex

@@ -8,6 +8,10 @@ defmodule Protobuf.Protoc.Context do
 
             ### All files scope
 
+            # All parsed comments from the source file (mapping from path to comment)
+            # %{"1.4.2" => "this is a comment", "1.5.2.4.2" => "more comment\ndetails"}
+            comments: %{},
+
             # Mapping from file name to (mapping from type name to metadata, like elixir type name)
             # %{"example.proto" => %{".example.FooMsg" => %{type_name: "Example.FooMsg"}}}
             global_type_mapping: %{},
@@ -42,7 +46,11 @@ defmodule Protobuf.Protoc.Context do
             include_docs?: false,
 
             # Elixirpb.FileOptions
-            custom_file_options: %{}
+            custom_file_options: %{},
+
+            # Current comment path. The locations are encoded into with a joining "."
+            # character. E.g. "4.2.3.0"
+            current_comment_path: ""
 
   @spec custom_file_options_from_file_desc(t(), Google.Protobuf.FileDescriptorProto.t()) :: t()
   def custom_file_options_from_file_desc(ctx, desc)
@@ -68,4 +76,17 @@ defmodule Protobuf.Protoc.Context do
         module_prefix: Map.get(custom_file_opts, :module_prefix)
     }
   end
+
+  @doc """
+  Appends a comment path to the current comment path.
+
+  ## Examples
+
+      iex> append_comment_path(%{current_comment_path: "4"}, "2.4")
+      %{current_comment_path: "4.2.4"}
+
+  """
+  def append_comment_path(ctx, path) do
+    %{ctx | current_comment_path: String.trim(ctx.current_comment_path <> "." <> path, ".")}
+  end
 end

lib/protobuf/protoc/generator.ex

@@ -36,13 +36,20 @@ defmodule Protobuf.Protoc.Generator do
     ctx =
       %Context{
         ctx
-        | syntax: syntax(desc.syntax),
+        | comments: Protobuf.Protoc.Generator.Comment.parse(desc),
+          syntax: syntax(desc.syntax),
           package: desc.package,
           dep_type_mapping: get_dep_type_mapping(ctx, desc.dependency, desc.name)
       }
       |> Protobuf.Protoc.Context.custom_file_options_from_file_desc(desc)
 
-    enum_defmodules = Enum.map(desc.enum_type, &Generator.Enum.generate(ctx, &1))
+    enum_defmodules =
+      desc.enum_type
+      |> Enum.with_index()
+      |> Enum.map(fn {enum, index} ->
+        {Context.append_comment_path(ctx, "5.#{index}"), enum}
+      end)
+      |> Enum.map(fn {ctx, enum} -> Generator.Enum.generate(ctx, enum) end)
 
     {nested_enum_defmodules, message_defmodules} =
       Generator.Message.generate_list(ctx, desc.message_type)
@@ -51,7 +58,14 @@ defmodule Protobuf.Protoc.Generator do
 
     service_defmodules =
       if "grpc" in ctx.plugins do
-        Enum.map(desc.service, &Generator.Service.generate(ctx, &1))
+        desc.service
+        |> Enum.with_index()
+        |> Enum.map(fn {service, index} ->
+          Generator.Service.generate(
+            Context.append_comment_path(ctx, "6.#{index}"),
+            service
+          )
+        end)
       else
         []
       end

lib/protobuf/protoc/generator/comment.ex

@@ -0,0 +1,58 @@
+defmodule Protobuf.Protoc.Generator.Comment do
+  @moduledoc false
+
+  alias Protobuf.Protoc.Context
+
+  @doc """
+  Parses comment information from `Google.Protobuf.FileDescriptorProto`
+  into a map with path keys.
+  """
+  @spec parse(Google.Protobuf.FileDescriptorProto.t()) :: %{optional(String.t()) => String.t()}
+  def parse(file_descriptor_proto) do
+    file_descriptor_proto
+    |> get_locations()
+    |> Enum.reject(&empty_comment?/1)
+    |> Map.new(fn location ->
+      {Enum.join(location.path, "."), format_comment(location)}
+    end)
+  end
+
+  defp get_locations(%{source_code_info: %{location: value}}) when is_list(value),
+    do: value
+
+  defp get_locations(_value), do: []
+
+  defp empty_comment?(%{leading_comments: value}) when not is_nil(value) and value != "",
+    do: false
+
+  defp empty_comment?(%{trailing_comments: value}) when not is_nil(value) and value != "",
+    do: false
+
+  defp empty_comment?(%{leading_detached_comments: value}), do: Enum.empty?(value)
+
+  defp format_comment(location) do
+    [location.leading_comments, location.trailing_comments | location.leading_detached_comments]
+    |> Enum.reject(&is_nil/1)
+    |> Enum.map(&String.replace(&1, ~r/^\s*\*/, "", global: true))
+    |> Enum.join("\n\n")
+    |> String.replace(~r/\n{3,}/, "\n")
+    |> String.trim()
+  end
+
+  @doc """
+  Finds a comment via the context. Returns an empty string if the
+  comment is not found or if `include_docs?` is set to false.
+  """
+  @spec get(Context.t()) :: String.t()
+  def get(%{include_docs?: false}), do: ""
+
+  def get(%{comments: comments, current_comment_path: path}),
+    do: get(comments, path)
+
+  @doc """
+  Finds a comment via a map of comments and a path. Returns an
+  empty string if the comment is not found
+  """
+  @spec get(%{optional(String.t()) => String.t()}, String.t()) :: String.t()
+  def get(comments, path), do: Map.get(comments, path, "")
+end

lib/protobuf/protoc/generator/enum.ex

@@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Enum do
   @moduledoc false
 
   alias Protobuf.Protoc.Context
+  alias Protobuf.Protoc.Generator.Comment
   alias Protobuf.Protoc.Generator.Util
 
   require EEx
@@ -34,6 +35,7 @@ defmodule Protobuf.Protoc.Generator.Enum do
 
     content =
       enum_template(
+        comment: Comment.get(ctx),
         module: msg_name,
         use_options: use_options,
         fields: desc.value,

lib/protobuf/protoc/generator/extension.ex

@@ -3,6 +3,7 @@ defmodule Protobuf.Protoc.Generator.Extension do
 
   alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto, FileDescriptorProto}
   alias Protobuf.Protoc.Context
+  alias Protobuf.Protoc.Generator.Comment
   alias Protobuf.Protoc.Generator.Util
 
   require EEx
@@ -29,7 +30,13 @@ defmodule Protobuf.Protoc.Generator.Extension do
 
     module_contents =
       Util.format(
-        extension_template(use_options: use_options, module: mod_name, extends: extensions)
+        extension_template(
+          comment: Comment.get(ctx),
+          use_options: use_options,
+          module: mod_name,
+          extends: extensions,
+          module_doc?: ctx.include_docs?
+        )
       )
 
     {mod_name, module_contents}
@@ -75,10 +82,15 @@ defmodule Protobuf.Protoc.Generator.Extension do
   end
 
   defp get_extensions_from_messages(%Context{} = ctx, use_options, descs) do
-    Enum.flat_map(descs, fn %DescriptorProto{} = desc ->
-      generate_module(ctx, use_options, desc) ++
+    descs
+    |> Enum.with_index()
+    |> Enum.flat_map(fn {desc, index} ->
+      generate_module(Context.append_comment_path(ctx, "7.#{index}"), use_options, desc) ++
         get_extensions_from_messages(
-          %Context{ctx | namespace: ctx.namespace ++ [Macro.camelize(desc.name)]},
+          %Context{
+            Context.append_comment_path(ctx, "6.#{index}")
+            | namespace: ctx.namespace ++ [Macro.camelize(desc.name)]
+          },
           use_options,
           desc.nested_type
         )
@@ -96,9 +108,11 @@ defmodule Protobuf.Protoc.Generator.Extension do
     module_contents =
       Util.format(
         extension_template(
+          comment: Comment.get(ctx),
           module: module_name,
           use_options: use_options,
-          extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = ""))
+          extends: Enum.map(desc.extension, &generate_extend_dsl(ctx, &1, _ns = "")),
+          module_doc?: ctx.include_docs?
         )
       )
 

lib/protobuf/protoc/generator/message.ex

@@ -4,6 +4,7 @@ defmodule Protobuf.Protoc.Generator.Message do
   alias Google.Protobuf.{DescriptorProto, FieldDescriptorProto}
 
   alias Protobuf.Protoc.Context
+  alias Protobuf.Protoc.Generator.Comment
   alias Protobuf.Protoc.Generator.Util
   alias Protobuf.Protoc.Generator.Enum, as: EnumGenerator
 
@@ -21,7 +22,10 @@ defmodule Protobuf.Protoc.Generator.Message do
            messages :: [{mod_name :: String.t(), contents :: String.t()}]}
   def generate_list(%Context{} = ctx, descs) when is_list(descs) do
     descs
-    |> Enum.map(fn desc -> generate(ctx, desc) end)
+    |> Enum.with_index()
+    |> Enum.map(fn {desc, index} ->
+      generate(Context.append_comment_path(ctx, "4.#{index}"), desc)
+    end)
     |> Enum.unzip()
   end
 
@@ -46,6 +50,7 @@ defmodule Protobuf.Protoc.Generator.Message do
       {msg_name,
        Util.format(
          message_template(
+           comment: Comment.get(ctx),
            module: msg_name,
            use_options: msg_opts_str(ctx, desc.options),
            oneofs: desc.oneof_decl,
@@ -61,11 +66,19 @@ defmodule Protobuf.Protoc.Generator.Message do
   end
 
   defp gen_nested_msgs(ctx, desc) do
-    Enum.map(desc.nested_type, fn msg_desc -> generate(ctx, msg_desc) end)
+    desc.nested_type
+    |> Enum.with_index()
+    |> Enum.map(fn {msg_desc, index} ->
+      generate(Context.append_comment_path(ctx, "3.#{index}"), msg_desc)
+    end)
   end
 
   defp gen_nested_enums(ctx, desc) do
-    Enum.map(desc.enum_type, fn enum_desc -> EnumGenerator.generate(ctx, enum_desc) end)
+    desc.enum_type
+    |> Enum.with_index()
+    |> Enum.map(fn {enum_desc, index} ->
+      EnumGenerator.generate(Context.append_comment_path(ctx, "4.#{index}"), enum_desc)
+    end)
   end
 
   defp gen_fields(syntax, fields) do
@@ -103,7 +116,15 @@ defmodule Protobuf.Protoc.Generator.Message do
     oneofs = get_real_oneofs(desc.oneof_decl)
 
     nested_maps = nested_maps(ctx, desc)
-    for field <- desc.field, do: get_field(ctx, field, nested_maps, oneofs)
+
+    for {field, index} <- Enum.with_index(desc.field) do
+      get_field(
+        Context.append_comment_path(ctx, "2.#{index}"),
+        field,
+        nested_maps,
+        oneofs
+      )
+    end
   end
 
   # Public and used by extensions.
@@ -137,6 +158,7 @@ defmodule Protobuf.Protoc.Generator.Message do
 
     %{
       name: field_desc.name,
+      comment: Comment.get(ctx),
       number: field_desc.number,
       label: label_name(field_desc.label),
       type: type,

lib/protobuf/protoc/generator/service.ex

@@ -2,6 +2,7 @@ defmodule Protobuf.Protoc.Generator.Service do
   @moduledoc false
 
   alias Protobuf.Protoc.Context
+  alias Protobuf.Protoc.Generator.Comment
   alias Protobuf.Protoc.Generator.Util
 
   require EEx
@@ -31,6 +32,7 @@ defmodule Protobuf.Protoc.Generator.Service do
     {mod_name,
      Util.format(
        service_template(
+         comment: Comment.get(ctx),
          module: mod_name,
          service_name: name,
          package: ctx.package,

lib/protobuf/protoc/generator/util.ex

@@ -86,6 +86,19 @@ defmodule Protobuf.Protoc.Generator.Util do
     |> IO.iodata_to_binary()
   end
 
+  @spec pad_comment(String.t(), non_neg_integer()) :: String.t()
+  def pad_comment(comment, size) do
+    padding = String.duplicate(" ", size)
+
+    comment
+    |> String.split("\n")
+    |> Enum.map(fn line ->
+      trimmed = String.trim_leading(line, " ")
+      padding <> trimmed
+    end)
+    |> Enum.join("\n")
+  end
+
   @spec version() :: String.t()
   def version do
     {:ok, value} = :application.get_key(:protobuf, :vsn)

mix.exs

@@ -149,15 +149,15 @@ defmodule Protobuf.Mixfile do
     proto_src = path_in_protobuf_source(["src"])
 
     protoc!(
-      "-I #{proto_src} -I src -I test/protobuf/protoc/proto",
+      "-I #{proto_src} -I src -I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
       "./generated",
       ["test/protobuf/protoc/proto/extension.proto"]
     )
 
     protoc!(
-      "-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my",
+      "-I test/protobuf/protoc/proto --elixir_opt=package_prefix=my,include_docs=true",
       "./generated",
-      ["test/protobuf/protoc/proto/test.proto"]
+      ["test/protobuf/protoc/proto/test.proto", "test/protobuf/protoc/proto/service.proto"]
     )
 
     protoc!(
@@ -168,7 +168,7 @@ defmodule Protobuf.Mixfile do
     )
 
     protoc!(
-      "-I test/protobuf/protoc/proto",
+      "-I test/protobuf/protoc/proto --elixir_opt=include_docs=true",
       "./generated",
       ["test/protobuf/protoc/proto/no_package.proto"]
     )
@@ -194,7 +194,7 @@ defmodule Protobuf.Mixfile do
       google/protobuf/test_messages_proto3.proto
     )
 
-    protoc!("-I \"#{proto_root}\"", "./generated", files)
+    protoc!("-I \"#{proto_root}\" --elixir_opt=include_docs=true", "./generated", files)
   end
 
   defp gen_conformance_protos(_args) do

priv/templates/enum.ex.eex

@@ -1,7 +1,14 @@
 defmodule <%= @module %> do
-  <%= unless @module_doc? do %>
+  <%= if @module_doc? do %>
+  <%= if @comment != "" do %>
+  @moduledoc """
+<%= Protobuf.Protoc.Generator.Util.pad_comment(@comment, 2) %>
+  """
+  <% end %>
+  <% else %>
   @moduledoc false
   <% end %>
+
   use Protobuf, <%= @use_options %>
 
   <%= if @descriptor_fun_body do %>

priv/templates/extension.ex.eex

@@ -1,5 +1,14 @@
 defmodule <%= @module %> do
+  <%= if @module_doc? do %>
+  <%= if @comment != "" do %>
+  @moduledoc """
+<%= Protobuf.Protoc.Generator.Util.pad_comment(@comment, 2) %>
+  """
+  <% end %>
+  <% else %>
   @moduledoc false
+  <% end %>
+
   use Protobuf, <%= @use_options %>
 
   <% if @extends == [], do: raise("Fuck! #{@module}") %>