dev: Get the test coverage HTML report to work again

.gitignore

@@ -23,7 +23,8 @@ gen_*.c
 gen_*.h
 wire/gen_*_csv
 cli/lightning-cli
-coverage
+coverage/raw
+coverage/index.html
 ccan/config.h
 __pycache__
 config.vars

Makefile

@@ -49,10 +49,6 @@ else
 DEV_CFLAGS=
 endif
 
-ifeq ($(COVERAGE),1)
-COVFLAGS = --coverage
-endif
-
 ifeq ($(CLANG_COVERAGE),1)
 COVFLAGS+=-fprofile-instr-generate -fcoverage-mapping
 endif
@@ -448,6 +444,12 @@ else
 PYTEST_OPTS += -x
 endif
 
+# No need to include the coverage Makefile if we're not going to run
+# those rules anyway.
+ifneq ($(CLANG_COVERAGE),0)
+	include coverage/Makefile
+endif
+
 check-units:
 
 check: check-units installcheck check-protos pytest
@@ -601,13 +603,6 @@ check-gen-updated:  $(CHECK_GEN_ALL)
 	@echo "Checking for generated files being changed by make"
 	git diff --exit-code HEAD
 
-coverage/coverage.info: check pytest
-	mkdir coverage || true
-	lcov --capture --directory . --output-file coverage/coverage.info
-
-coverage: coverage/coverage.info
-	genhtml coverage/coverage.info --output-directory coverage
-
 # We make libwallycore.la a dependency, so that it gets built normally, without ncc.
 # Ncc can't handle the libwally source code (yet).
 ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la

contrib/prepare-code-coverage-artifact.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+
+"""This script was taken from the LLVM project, and is copyrighted
+under the Apache License 2.0
+
+"""
+
+from __future__ import print_function
+
+'''Prepare a code coverage artifact.
+
+- Collate raw profiles into one indexed profile.
+- Generate html reports for the given binaries.
+
+Caution: The positional arguments to this script must be specified before any
+optional arguments, such as --restrict.
+'''
+
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+def merge_raw_profiles(host_llvm_profdata, profile_data_dir, preserve_profiles):
+    print(':: Merging raw profiles...', end='')
+    sys.stdout.flush()
+    raw_profiles = glob.glob(os.path.join(profile_data_dir, '*.profraw'))
+    manifest_path = os.path.join(profile_data_dir, 'profiles.manifest')
+    profdata_path = os.path.join(profile_data_dir, 'Coverage.profdata')
+    with open(manifest_path, 'w') as manifest:
+        manifest.write('\n'.join(raw_profiles))
+    subprocess.check_call([host_llvm_profdata, 'merge', '-sparse', '--failure-mode=all', '-f',
+                           manifest_path, '-o', profdata_path])
+    if not preserve_profiles:
+        for raw_profile in raw_profiles:
+            os.remove(raw_profile)
+    os.remove(manifest_path)
+    print('Done!')
+    return profdata_path
+
+def prepare_html_report(host_llvm_cov, profile, report_dir, binaries,
+                        restricted_dirs):
+    print(':: Preparing html report for {0}...'.format(binaries), end='')
+    sys.stdout.flush()
+    objects = []
+    for i, binary in enumerate(binaries):
+        if i == 0:
+            objects.append(binary)
+        else:
+            objects.extend(('-object', binary))
+    invocation = [host_llvm_cov, 'show'] + objects + ['-format', 'html',
+                  '-instr-profile', profile, '-o', report_dir,
+                  '-show-line-counts-or-regions', '-Xdemangler', 'c++filt',
+                  '-Xdemangler', '-n'] + restricted_dirs
+    subprocess.check_call(invocation)
+    with open(os.path.join(report_dir, 'summary.txt'), 'wb') as Summary:
+        subprocess.check_call([host_llvm_cov, 'report'] + objects +
+                               ['-instr-profile', profile] + restricted_dirs,
+                               stdout=Summary)
+    print('Done!')
+
+def prepare_html_reports(host_llvm_cov, profdata_path, report_dir, binaries,
+                         unified_report, restricted_dirs):
+    if unified_report:
+        prepare_html_report(host_llvm_cov, profdata_path, report_dir, binaries,
+                            restricted_dirs)
+    else:
+        for binary in binaries:
+            binary_report_dir = os.path.join(report_dir,
+                                             os.path.basename(binary))
+            prepare_html_report(host_llvm_cov, profdata_path, binary_report_dir,
+                                [binary], restricted_dirs)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('host_llvm_profdata', help='Path to llvm-profdata')
+    parser.add_argument('host_llvm_cov', help='Path to llvm-cov')
+    parser.add_argument('profile_data_dir',
+                       help='Path to the directory containing the raw profiles')
+    parser.add_argument('report_dir',
+                       help='Path to the output directory for html reports')
+    parser.add_argument('binaries', metavar='B', type=str, nargs='*',
+                       help='Path to an instrumented binary')
+    parser.add_argument('--only-merge', action='store_true',
+                        help='Only merge raw profiles together, skip report '
+                             'generation')
+    parser.add_argument('--preserve-profiles',
+                       help='Do not delete raw profiles', action='store_true')
+    parser.add_argument('--use-existing-profdata',
+                       help='Specify an existing indexed profile to use')
+    parser.add_argument('--unified-report', action='store_true',
+                       help='Emit a unified report for all binaries')
+    parser.add_argument('--restrict', metavar='R', type=str, nargs='*',
+                       default=[],
+                       help='Restrict the reporting to the given source paths'
+                   ' (must be specified after all other positional arguments)')
+    args = parser.parse_args()
+
+    if args.use_existing_profdata and args.only_merge:
+        print('--use-existing-profdata and --only-merge are incompatible')
+        exit(1)
+
+    if args.use_existing_profdata:
+        profdata_path = args.use_existing_profdata
+    else:
+        profdata_path = merge_raw_profiles(args.host_llvm_profdata,
+                                           args.profile_data_dir,
+                                           args.preserve_profiles)
+
+    if not len(args.binaries):
+        print('No binaries specified, no work to do!')
+        exit(1)
+
+    if not args.only_merge:
+        prepare_html_reports(args.host_llvm_cov, profdata_path, args.report_dir,
+                            args.binaries, args.unified_report, args.restrict)

doc/developers-guide/coverage.md

@@ -0,0 +1,30 @@
+# Test Coverage
+
+> Coverage isn't everything, but it can tell you were you missed a thing.
+
+We use LLVM's [Source-Based Code Coverage][sbcc] support to instrument
+the code at compile time. This instrumentation then emits coverage
+files (`profraw`), which can then be aggregated via `llvm-profdata`
+into a single `profdata` file, and from there a variety of tools can
+be used to inspect coverage.
+
+The most common use is to generate an HTML report for all binaries
+under test. CLN being a multi-process system has a number of binaries,
+sharing some source code. To simplify the aggregation of data and
+generation of the report split per source file, we use the
+`prepare-code-coverage-artifact.py` ([`pcca.py`][pcca]) script from
+the LLVM project.
+
+## Conventions
+
+The `tests/fixtures.py` sets the `LLVM_PROFILE_FILE` environment
+variable, indicating that the `profraw` files ought to be stores in
+`coverage/raw`. Processing the file then uses [`pcca.py`][pcca] to
+aggregate the raw files, into a data file, and then generate a
+per-source-file coverage report.
+
+This report is then published [here][report]
+
+[sbcc]: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
+[pcca]: https://github.com/ElementsProject/lightning/tree/master/contrib/prepare-code-coverage-artifact.py
+[report]: https://cdecker.github.io/lightning/coverage

tests/fixtures.py

@@ -1,5 +1,6 @@
 from utils import TEST_NETWORK, VALGRIND  # noqa: F401,F403
 from pyln.testing.fixtures import directory, test_base_dir, test_name, chainparams, node_factory, bitcoind, teardown_checks, db_provider, executor, setup_logging, jsonschemas  # noqa: F401,F403
+from pyln.testing import fixtures
 from pyln.testing import utils
 from utils import COMPAT
 from pathlib import Path
@@ -10,7 +11,16 @@
 
 
 @pytest.fixture
-def node_cls():
+def node_cls(test_name: str):
+    # We always set the LLVM coverage destination, just in case
+    # `lightningd` was compiled with the correct instrumentation
+    # flags. This creates a `coverage` directory in the repository
+    # and puts all the files in it.
+    repo_root = Path(__file__).parent.parent
+    os.environ['LLVM_PROFILE_FILE'] = str(
+        repo_root / "coverage" / "raw" / f"{test_name}.%p.profraw"
+    )
+
     return LightningNode