Teach mull to extract coverage information automatically

docs/command-line/generated/mull-runner-cli-options.rst

@@ -36,3 +36,7 @@
 
 --ld-search-path directory		Library search path
 
+--coverage-info string		Path to the coverage info file (LLVM's profdata)
+
+--debug-coverage		Print coverage ranges
+

include/mull/Toolchain/Runner.h

@@ -3,6 +3,7 @@
 #include "mull/ExecutionResult.h"
 #include <optional>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 namespace mull {
@@ -13,8 +14,8 @@ class Runner {
 public:
   explicit Runner(Diagnostics &diagnostics);
   ExecutionResult runProgram(const std::string &program, const std::vector<std::string> &arguments,
-                             const std::vector<std::string> &environment, long long int timeout,
-                             bool captureOutput,
+                             const std::unordered_map<std::string, std::string> &environment,
+                             long long int timeout, bool captureOutput,
                              std::optional<std::string> optionalWorkingDirectory);
 
 private:

lib/Parallelization/Tasks/MutantExecutionTask.cpp

@@ -2,10 +2,10 @@
 
 #include "mull/Config/Configuration.h"
 #include "mull/Diagnostics/Diagnostics.h"
+#include "mull/ExecutionResult.h"
 #include "mull/Parallelization/Progress.h"
 #include "mull/SourceLocation.h"
 #include "mull/Toolchain/Runner.h"
-#include "mull/ExecutionResult.h"
 
 #include <sstream>
 
@@ -29,7 +29,7 @@ void MutantExecutionTask::operator()(iterator begin, iterator end, Out &storage,
     if (mutant->isCovered()) {
       result = runner.runProgram(executable,
                                  extraArgs,
-                                 { mutant->getIdentifier() },
+                                 { { mutant->getIdentifier(), "1" } },
                                  std::max(30LL, baseline.runningTime * 10),
                                  configuration.captureMutantOutput,
                                  std::nullopt);

lib/Toolchain/Runner.cpp

@@ -29,17 +29,11 @@ Runner::Runner(Diagnostics &diagnostics) : diagnostics(diagnostics) {}
 
 ExecutionResult Runner::runProgram(const std::string &program,
                                    const std::vector<std::string> &arguments,
-                                   const std::vector<std::string> &environment,
+                                   const std::unordered_map<std::string, std::string> &environment,
                                    long long int timeout, bool captureOutput,
                                    std::optional<std::string> optionalWorkingDirectory) {
-  std::vector<std::pair<std::string, std::string>> env;
-  env.reserve(environment.size());
-  for (auto &e : environment) {
-    env.emplace_back(e, "1");
-  }
-
   reproc::options options;
-  options.env.extra = reproc::env(env);
+  options.env.extra = reproc::env(environment);
   options.redirect.err.type = reproc::redirect::type::pipe;
   options.stop.first.action = reproc::stop::kill;
   options.stop.first.timeout = std::chrono::milliseconds(100);

tests-lit/tests/coverage/01/main.c

@@ -10,13 +10,15 @@ int main(int argc, char **argv) {
 }
 
 // clang-format off
-// RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s.exe
+// RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -o %s.exe
 // RUN: unset TERM; %mull_runner %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
 // CHECK-NO-COVERAGE:[info] Survived mutants (1/1):
 // CHECK-NO-COVERAGE:{{^.*}}main.c:2:5: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-NO-COVERAGE:[info] Mutation score: 0%
 
 // RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s-cov.exe
+// RUN: unset TERM; %mull_runner %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
+
 // RUN: env LLVM_PROFILE_FILE=%s.profraw %s-cov.exe
 // RUN: %llvm_profdata merge %s.profraw -o %s.profdata
 // RUN: unset TERM; %mull_runner -coverage-info=%s.profdata %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE

tests-lit/tests/coverage/02/main.c

@@ -8,6 +8,7 @@ int main() {
 
 // clang-format off
 // RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s.exe
+// RUN: unset TERM; %mull_runner -ide-reporter-show-killed -include-not-covered %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines
 // RUN: env LLVM_PROFILE_FILE=%s.profraw %s.exe
 // RUN: %llvm_profdata merge %s.profraw -o %s.profdata
 // RUN: unset TERM; %mull_runner -ide-reporter-show-killed -include-not-covered -coverage-info %s.profdata %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines

tests-lit/tests/coverage/partial-covered-functions/main.c

@@ -22,9 +22,7 @@ int main(){
   return 0;
 }
 // clang-format off
-// RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s.exe
-// RUN: env LLVM_PROFILE_FILE=%s.profraw %s.exe
-// RUN: %llvm_profdata merge %s.profraw -o %s.profdata
+// RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -o %s.exe
 // RUN: unset TERM; %mull_runner %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-NO-COVERAGE
 // CHECK-NO-COVERAGE:[info] Survived mutants (4/4):
 // CHECK-NO-COVERAGE:{{^.*}}main.c:5:16: warning: Survived: Replaced + with - [cxx_add_to_sub]
@@ -33,7 +31,11 @@ int main(){
 // CHECK-NO-COVERAGE:{{^.*}}main.c:14:25: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-NO-COVERAGE:[info] Mutation score: 0%
 
-// RUN: unset TERM; %mull_runner -coverage-info %s.profdata -include-not-covered %s.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
+// RUN: %clang_cc %sysroot %s %pass_mull_ir_frontend -g -fprofile-instr-generate -fcoverage-mapping -o %s-cov.exe
+// RUN: unset TERM; %mull_runner -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
+// RUN: env LLVM_PROFILE_FILE=%s.profraw %s-cov.exe
+// RUN: %llvm_profdata merge %s.profraw -o %s.profdata
+// RUN: unset TERM; %mull_runner -coverage-info %s.profdata -include-not-covered %s-cov.exe 2>&1 | %filecheck %s --dump-input=fail --strict-whitespace --match-full-lines --check-prefix=CHECK-COVERAGE
 // CHECK-COVERAGE:[info] Survived mutants (2/4):
 // CHECK-COVERAGE:{{^.*}}main.c:10:11: warning: Survived: Replaced + with - [cxx_add_to_sub]
 // CHECK-COVERAGE:{{^.*}}main.c:14:21: warning: Survived: Replaced + with - [cxx_add_to_sub]

tools/mull-runner/CMakeLists.txt

@@ -4,6 +4,8 @@ set (SOURCES
   ${CMAKE_CURRENT_LIST_DIR}/DynamicLibraries.cpp
   ${CMAKE_CURRENT_LIST_DIR}/../CLIOptions/CLIOptions.cpp
   ${CMAKE_CURRENT_LIST_DIR}/CoverageChecker.cpp
+  ${CMAKE_CURRENT_LIST_DIR}/MergeInstProfile.cpp
+
 )
 
 add_mull_executable(

tools/mull-runner/DynamicLibraries.cpp

@@ -169,3 +169,34 @@ std::vector<std::string> mull::getDynamicLibraryDependencies(mull::Diagnostics &
   }
   return libraries;
 }
+
+bool mull::hasCoverage(mull::Diagnostics &diagnostics, const std::string &path) {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> maybeBuffer =
+      llvm::MemoryBuffer::getFile(path);
+  if (!maybeBuffer) {
+    diagnostics.error("Cannot read executable: "s + maybeBuffer.getError().message() + ": " + path);
+  }
+  llvm::MemoryBuffer *buffer = maybeBuffer->get();
+  llvm::Expected<std::unique_ptr<llvm::object::ObjectFile>> maybeObject =
+      llvm::object::ObjectFile::createObjectFile(buffer->getMemBufferRef());
+  if (!maybeObject) {
+    llvm::Error error = maybeObject.takeError();
+    /// On older versions of macOS we fail to load certain system libraries because they are fat
+    /// libraries On newer versions of macOS we never reach this line because the system libraries
+    /// live in cache and cannot be read from FS This should be an error, but we relax it to a
+    /// warning to not fail on macOS
+    /// TODO: we should probably add support for universal binaries at some point
+    /// https://github.com/mull-project/mull/issues/932
+    diagnostics.warning("Skipping. Executable is not an object file: "s +
+                        llvm::toString(std::move(error)) + ": " + path);
+    return false;
+  }
+
+  for (auto &section : (*maybeObject)->sections()) {
+    if (getSectionName(section).endswith("llvm_covmap")) {
+      return true;
+    }
+  }
+
+  return false;
+}

tools/mull-runner/DynamicLibraries.h

@@ -13,4 +13,7 @@ void resolveLibraries(mull::Diagnostics &diagnostics, std::vector<std::string> &
 
 std::vector<std::string> getDynamicLibraryDependencies(mull::Diagnostics &diagnostics,
                                                        const std::string &executablePath);
-} // namespace mull
+
+bool hasCoverage(mull::Diagnostics &diagnostics, const std::string &path);
+
+} // namespace mull

tools/mull-runner/MergeInstProfile.cpp

@@ -0,0 +1,30 @@
+#include "MergeInstProfile.h"
+#include "mull/Diagnostics/Diagnostics.h"
+#include <llvm/ProfileData/InstrProfReader.h>
+#include <llvm/ProfileData/InstrProfWriter.h>
+#include <llvm/Support/raw_ostream.h>
+
+using namespace std::string_literals;
+
+bool mull::mergeRawInstProfile(Diagnostics &diagnostics, const std::string &input,
+                               const std::string &output) {
+  auto maybeReader = llvm::InstrProfReader::create(input);
+  if (!maybeReader) {
+    diagnostics.warning("cannot read raw profile data: "s +
+                        llvm::toString(maybeReader.takeError()));
+    return false;
+  }
+  llvm::InstrProfWriter writer;
+  auto &reader = maybeReader.get();
+  for (auto &i : *reader) {
+    writer.addRecord(std::move(i), [](llvm::Error error) {});
+  }
+  std::error_code ec;
+  llvm::raw_fd_ostream out(output, ec);
+  if (ec) {
+    diagnostics.warning("cannot save indexed profile data: "s + ec.message());
+    return false;
+  }
+  writer.write(out);
+  return true;
+}

tools/mull-runner/MergeInstProfile.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+
+namespace mull {
+
+class Diagnostics;
+
+bool mergeRawInstProfile(Diagnostics &diagnostics, const std::string &input,
+                         const std::string &output);
+
+} // namespace mull

tools/mull-runner/MutantExtractor.cpp

@@ -2,7 +2,6 @@
 #include <iostream>
 #include <llvm/Object/ObjectFile.h>
 #include <sstream>
-#include <unordered_map>
 #include <unordered_set>
 
 using namespace mull;

tools/mull-runner/mull-runner.cpp

@@ -1,5 +1,6 @@
 #include "CoverageChecker.h"
 #include "DynamicLibraries.h"
+#include "MergeInstProfile.h"
 #include "MutantExtractor.h"
 #include "mull-runner-cli.h"
 #include "mull/Config/Configuration.h"
@@ -12,6 +13,7 @@
 #include "mull/Version.h"
 
 #include <llvm/Support/FileSystem.h>
+#include <llvm/Support/Path.h>
 
 #include <memory>
 #include <unistd.h>
@@ -132,7 +134,27 @@ int main(int argc, char **argv) {
   mull::MutantExtractor mutantExtractor(diagnostics);
   std::vector<std::unique_ptr<mull::Mutant>> mutants =
       mutantExtractor.extractMutants(mutantHolders);
+  std::string coverageInfo = tool::CoverageInfo.getValue();
 
+  std::string rawCoverageData;
+
+  std::unordered_map<std::string, std::string> env;
+  if (coverageInfo.empty() && mull::hasCoverage(diagnostics, configuration.executable)) {
+    llvm::SmallString<PATH_MAX> rawPath;
+    llvm::sys::fs::getPotentiallyUniqueTempFileName("mull", "raw-coverage", rawPath);
+    rawCoverageData = rawPath.str().str();
+
+    llvm::SmallString<PATH_MAX> indexedPath;
+    llvm::sys::fs::getPotentiallyUniqueTempFileName("mull", "indexed-coverage", indexedPath);
+    coverageInfo = indexedPath.str().str();
+
+    if (configuration.debug.coverage) {
+      llvm::errs() << "rawCoverageFile: " << rawCoverageData << "\n";
+      llvm::errs() << "indexedCoverageFile: " << coverageInfo << "\n";
+    }
+
+    env["LLVM_PROFILE_FILE"] = rawCoverageData;
+  }
   mull::Runner runner(diagnostics);
   mull::SingleTaskExecutor singleTask(diagnostics);
   /// On macOS, sometimes newly compiled programs take more time to execute for the first run
@@ -142,7 +164,7 @@ int main(int argc, char **argv) {
   singleTask.execute("Warm up run", [&]() {
     warmUpResult = runner.runProgram(testProgram,
                                      extraArgs,
-                                     {},
+                                     env,
                                      configuration.timeout,
                                      configuration.captureMutantOutput,
                                      std::nullopt);
@@ -151,9 +173,15 @@ int main(int argc, char **argv) {
     diagnostics.warning(warmUpResult.debugDescription());
   }
 
+  if (!rawCoverageData.empty()) {
+    singleTask.execute("Extracting coverage information", [&]() {
+      mull::mergeRawInstProfile(diagnostics, rawCoverageData, coverageInfo);
+      llvm::sys::fs::remove(rawCoverageData);
+    });
+  }
+
   std::vector<std::unique_ptr<mull::Mutant>> filteredMutants;
-  mull::CoverageChecker coverage(
-      configuration, diagnostics, tool::CoverageInfo.getValue(), mutantHolders);
+  mull::CoverageChecker coverage(configuration, diagnostics, coverageInfo, mutantHolders);
   for (auto &mutant : mutants) {
     bool covered = coverage.covered(mutant.get());
     mutant->setCovered(covered);