Add test coverage support (#1324)

* WIP coverage support

* Make it work

* Regenerate documentation

* Gate env vars on coverage being enabled

* Add coverage targets

* Allow 2 actions when generating coverage

Co-authored-by: Krasimir Georgiev <krasimir@google.com>

.bazelci/presubmit.yml

@@ -25,6 +25,7 @@ tasks:
   ubuntu2004:
     build_targets: *default_linux_targets
     test_targets: *default_linux_targets
+    coverage_targets: *default_linux_targets
   rbe_ubuntu1604:
     shell_commands:
       - sed -i 's/^# load("@bazelci_rules/load("@bazelci_rules/' WORKSPACE.bazel
@@ -39,6 +40,7 @@ tasks:
   macos:
     build_targets: *default_macos_targets
     test_targets: *default_macos_targets
+    coverage_targets: *default_macos_targets
     build_flags: *aspects_flags
   windows:
     build_flags:

docs/flatten.md

@@ -1062,9 +1062,9 @@ Run the test with `bazel test //hello_lib:greeting_test`.
 
 <pre>
 rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
-               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>,
-               <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>,
-               <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
+               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_cov">llvm_cov</a>, <a href="#rust_toolchain-llvm_profdata">llvm_profdata</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>,
+               <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>,
+               <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
 </pre>
 
 Declares a Rust toolchain for use.
@@ -1123,6 +1123,8 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r
 | <a id="rust_toolchain-dylib_ext"></a>dylib_ext |  The extension for dynamic libraries created from rustc.   | String | required |  |
 | <a id="rust_toolchain-env"></a>env |  Environment variables to set in actions.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
 | <a id="rust_toolchain-exec_triple"></a>exec_triple |  The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations   | String | required |  |
+| <a id="rust_toolchain-llvm_cov"></a>llvm_cov |  The location of the <code>llvm-cov</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="rust_toolchain-llvm_profdata"></a>llvm_profdata |  The location of the <code>llvm-profdata</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="rust_toolchain-llvm_tools"></a>llvm_tools |  LLVM tools that are shipped with the Rust toolchain.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="rust_toolchain-opt_level"></a>opt_level |  Rustc optimization levels.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
 | <a id="rust_toolchain-os"></a>os |  The operating system for the current toolchain   | String | required |  |

docs/rust_repositories.md

@@ -35,9 +35,9 @@ A dedicated filegroup-like rule for Rust stdlib artifacts.
 
 <pre>
 rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
-               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>,
-               <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>,
-               <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
+               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-env">env</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_cov">llvm_cov</a>, <a href="#rust_toolchain-llvm_profdata">llvm_profdata</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>,
+               <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>,
+               <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
 </pre>
 
 Declares a Rust toolchain for use.
@@ -96,6 +96,8 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r
 | <a id="rust_toolchain-dylib_ext"></a>dylib_ext |  The extension for dynamic libraries created from rustc.   | String | required |  |
 | <a id="rust_toolchain-env"></a>env |  Environment variables to set in actions.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
 | <a id="rust_toolchain-exec_triple"></a>exec_triple |  The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations   | String | required |  |
+| <a id="rust_toolchain-llvm_cov"></a>llvm_cov |  The location of the <code>llvm-cov</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="rust_toolchain-llvm_profdata"></a>llvm_profdata |  The location of the <code>llvm-profdata</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="rust_toolchain-llvm_tools"></a>llvm_tools |  LLVM tools that are shipped with the Rust toolchain.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
 | <a id="rust_toolchain-opt_level"></a>opt_level |  Rustc optimization levels.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
 | <a id="rust_toolchain-os"></a>os |  The operating system for the current toolchain   | String | required |  |

rust/private/repository_utils.bzl

@@ -137,6 +137,35 @@ def BUILD_for_clippy(target_triple):
     system = triple_to_system(target_triple)
     return _build_file_for_clippy_template.format(binary_ext = system_to_binary_ext(system))
 
+_build_file_for_llvm_tools = """\
+filegroup(
+    name = "llvm_cov_bin",
+    srcs = ["lib/rustlib/{target_triple}/bin/llvm-cov{binary_ext}"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "llvm_profdata_bin",
+    srcs = ["lib/rustlib/{target_triple}/bin/llvm-profdata{binary_ext}"],
+    visibility = ["//visibility:public"],
+)
+"""
+
+def BUILD_for_llvm_tools(target_triple):
+    """Emits a BUILD file the llvm-tools binaries.
+
+    Args:
+        target_triple (str): The triple of the target platform
+
+    Returns:
+        str: The contents of a BUILD file
+    """
+    system = triple_to_system(target_triple)
+    return _build_file_for_llvm_tools.format(
+        binary_ext = system_to_binary_ext(system),
+        target_triple = target_triple,
+    )
+
 _build_file_for_stdlib_template = """\
 load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup")
 
@@ -189,6 +218,8 @@ rust_toolchain(
     rustfmt = {rustfmt_label},
     cargo = "@{workspace_name}//:cargo",
     clippy_driver = "@{workspace_name}//:clippy_driver_bin",
+    llvm_cov = {llvm_cov_label},
+    llvm_profdata = {llvm_profdata_label},
     rustc_lib = "@{workspace_name}//:rustc_lib",
     rustc_srcs = {rustc_srcs},
     binary_ext = "{binary_ext}",
@@ -211,6 +242,7 @@ def BUILD_for_rust_toolchain(
         include_rustc_srcs,
         default_edition,
         include_rustfmt,
+        include_llvm_tools,
         stdlib_linkflags = None):
     """Emits a toolchain declaration to match an existing compiler and stdlib.
 
@@ -222,6 +254,7 @@ def BUILD_for_rust_toolchain(
         include_rustc_srcs (bool, optional): Whether to download rustc's src code. This is required in order to use rust-analyzer support. Defaults to False.
         default_edition (str): Default Rust edition.
         include_rustfmt (bool): Whether rustfmt is present in the toolchain.
+        include_llvm_tools (bool): Whether llvm-tools are present in the toolchain.
         stdlib_linkflags (list, optional): Overriden flags needed for linking to rust
                                            stdlib, akin to BAZEL_LINKLIBS. Defaults to
                                            None.
@@ -240,6 +273,11 @@ def BUILD_for_rust_toolchain(
     rustfmt_label = "None"
     if include_rustfmt:
         rustfmt_label = "\"@{workspace_name}//:rustfmt_bin\"".format(workspace_name = workspace_name)
+    llvm_cov_label = "None"
+    llvm_profdata_label = "None"
+    if include_llvm_tools:
+        llvm_cov_label = "\"@{workspace_name}//:llvm_cov_bin\"".format(workspace_name = workspace_name)
+        llvm_profdata_label = "\"@{workspace_name}//:llvm_profdata_bin\"".format(workspace_name = workspace_name)
 
     return _build_file_for_rust_toolchain_template.format(
         toolchain_name = name,
@@ -254,6 +292,8 @@ def BUILD_for_rust_toolchain(
         exec_triple = exec_triple,
         target_triple = target_triple,
         rustfmt_label = rustfmt_label,
+        llvm_cov_label = llvm_cov_label,
+        llvm_profdata_label = llvm_profdata_label,
     )
 
 _build_file_for_toolchain_template = """\
@@ -369,12 +409,13 @@ filegroup(
 )""",
     )
 
-def load_rust_stdlib(ctx, target_triple):
+def load_rust_stdlib(ctx, target_triple, include_llvm_tools):
     """Loads a rust standard library and yields corresponding BUILD for it
 
     Args:
         ctx (repository_ctx): A repository_ctx.
         target_triple (str): The rust-style target triple of the tool
+        include_llvm_tools (bool): Whether to include LLVM tools in the toolchain
 
     Returns:
         str: The BUILD file contents for this stdlib, and a toolchain decl to match
@@ -408,6 +449,7 @@ def load_rust_stdlib(ctx, target_triple):
         workspace_name = ctx.attr.name,
         default_edition = ctx.attr.edition,
         include_rustfmt = not (not ctx.attr.rustfmt_version),
+        include_llvm_tools = include_llvm_tools,
     )
 
     return stdlib_build_file + toolchain_build_file
@@ -449,6 +491,8 @@ def load_llvm_tools(ctx, target_triple):
         version = ctx.attr.version,
     )
 
+    return BUILD_for_llvm_tools(target_triple)
+
 def check_version_valid(version, iso_date, param_prefix = ""):
     """Verifies that the provided rust version and iso_date make sense.
 

rust/private/rust.bzl

@@ -466,6 +466,9 @@ def _rust_test_common(ctx, toolchain, output):
         getattr(ctx.attr, "env", {}),
         data,
     )
+    if ctx.configuration.coverage_enabled:
+        env["RUST_LLVM_COV"] = toolchain.llvm_cov.path
+        env["RUST_LLVM_PROFDATA"] = toolchain.llvm_profdata.path
     providers.append(testing.TestEnvironment(env))
 
     return providers
@@ -649,6 +652,11 @@ _common_attrs = {
     "_cc_toolchain": attr.label(
         default = "@bazel_tools//tools/cpp:current_cc_toolchain",
     ),
+    "_collect_cc_coverage": attr.label(
+        default = "//util:collect_coverage",
+        executable = True,
+        cfg = "exec",
+    ),
     "_error_format": attr.label(default = "//:error_format"),
     "_extra_exec_rustc_flags": attr.label(default = "//:extra_exec_rustc_flags"),
     "_extra_rustc_flags": attr.label(default = "//:extra_rustc_flags"),

rust/private/rustc.bzl

@@ -805,6 +805,9 @@ def construct_arguments(
         rustc_flags.add("--extern")
         rustc_flags.add("proc_macro")
 
+    if ctx.configuration.coverage_enabled:
+        rustc_flags.add("--codegen=instrument-coverage")
+
     # Make bin crate data deps available to tests.
     for data in getattr(attr, "data", []):
         if rust_common.crate_info in data:
@@ -993,8 +996,12 @@ def rustc_compile_action(
             ),
         )
 
+    coverage_runfiles = []
+    if toolchain.llvm_cov and ctx.configuration.coverage_enabled and crate_info.is_test:
+        coverage_runfiles = [toolchain.llvm_cov, toolchain.llvm_profdata]
+
     runfiles = ctx.runfiles(
-        files = getattr(ctx.files, "data", []),
+        files = getattr(ctx.files, "data", []) + coverage_runfiles,
         collect_data = True,
     )
 
@@ -1009,6 +1016,12 @@ def rustc_compile_action(
             runfiles = runfiles,
             executable = crate_info.output if crate_info.type == "bin" or crate_info.is_test or out_binary else None,
         ),
+        coverage_common.instrumented_files_info(
+            ctx,
+            dependency_attributes = ["deps", "crate"],
+            extensions = ["rs"],
+            source_attributes = ["srcs"],
+        ),
     ]
 
     if crate_info.type in ["staticlib", "cdylib"]:

rust/repositories.bzl

@@ -173,11 +173,12 @@ def _rust_toolchain_repository_impl(ctx):
         build_components.append(load_rustfmt(ctx))
 
     # Rust 1.45.0 and nightly builds after 2020-05-22 need the llvm-tools gzip to get the libLLVM dylib
-    if ctx.attr.version >= "1.45.0" or (ctx.attr.version == "nightly" and ctx.attr.iso_date > "2020-05-22"):
-        load_llvm_tools(ctx, ctx.attr.exec_triple)
+    include_llvm_tools = ctx.attr.version >= "1.45.0" or (ctx.attr.version == "nightly" and ctx.attr.iso_date > "2020-05-22")
+    if include_llvm_tools:
+        build_components.append(load_llvm_tools(ctx, ctx.attr.exec_triple))
 
     for target_triple in [ctx.attr.exec_triple] + ctx.attr.extra_target_triples:
-        build_components.append(load_rust_stdlib(ctx, target_triple))
+        build_components.append(load_rust_stdlib(ctx, target_triple, include_llvm_tools))
 
         # extra_target_triples contains targets such as wasm, which don't have rustc_dev components
         if ctx.attr.dev_components and target_triple not in ctx.attr.extra_target_triples:

rust/toolchain.bzl

@@ -495,6 +495,8 @@ def _rust_toolchain_impl(ctx):
         env = ctx.attr.env,
         exec_triple = ctx.attr.exec_triple,
         libstd_and_allocator_ccinfo = _make_libstd_and_allocator_ccinfo(ctx, rust_std, ctx.attr.allocator_library),
+        llvm_cov = ctx.file.llvm_cov,
+        llvm_profdata = ctx.file.llvm_profdata,
         make_variables = platform_common.TemplateVariableInfo(make_variables),
         os = ctx.attr.os,
         rust_doc = sysroot.rustdoc,
@@ -569,6 +571,16 @@ rust_toolchain = rule(
             ),
             mandatory = True,
         ),
+        "llvm_cov": attr.label(
+            doc = "The location of the `llvm-cov` binary. Can be a direct source or a filegroup containing one item.",
+            allow_single_file = True,
+            cfg = "exec",
+        ),
+        "llvm_profdata": attr.label(
+            doc = "The location of the `llvm-profdata` binary. Can be a direct source or a filegroup containing one item.",
+            allow_single_file = True,
+            cfg = "exec",
+        ),
         "llvm_tools": attr.label(
             doc = "LLVM tools that are shipped with the Rust toolchain.",
             allow_files = True,

test/unit/native_deps/native_deps_test.bzl

@@ -48,7 +48,10 @@ def _staticlib_has_native_libs_test_impl(ctx):
 def _proc_macro_has_native_libs_test_impl(ctx):
     env = analysistest.begin(ctx)
     tut = analysistest.target_under_test(env)
-    asserts.equals(env, 1, len(tut.actions))
+    if ctx.configuration.coverage_enabled:
+        asserts.equals(env, 2, len(tut.actions))
+    else:
+        asserts.equals(env, 1, len(tut.actions))
     action = tut.actions[0]
     assert_argv_contains_prefix_suffix(env, action, "-Lnative=", "/native_deps")
     assert_argv_contains(env, action, "--crate-type=proc-macro")

util/BUILD.bazel

@@ -16,3 +16,9 @@ filegroup(
     ],
     visibility = ["//:__subpackages__"],
 )
+
+filegroup(
+    name = "collect_coverage",
+    srcs = ["collect_coverage.sh"],
+    visibility = ["//visibility:public"],
+)

util/collect_coverage.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ -n "${VERBOSE_COVERAGE:-}" ]]; then
+  set -x
+fi
+
+readonly profdata_file=$COVERAGE_DIR/coverage.profdata
+
+"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_PROFDATA" \
+  merge \
+  --sparse "$COVERAGE_DIR"/*.profraw \
+  -output "$profdata_file"
+
+"$RUNFILES_DIR/$TEST_WORKSPACE/$RUST_LLVM_COV" \
+  export \
+  -format=lcov \
+  -instr-profile "$profdata_file" \
+  -ignore-filename-regex='.*external/.+' \
+  -ignore-filename-regex='/tmp/.+' \
+  -path-equivalence="$ROOT",. \
+  "$RUNFILES_DIR/$TEST_WORKSPACE/$TEST_BINARY" \
+  | sed 's#/proc/self/cwd/##' > "$COVERAGE_OUTPUT_FILE"