From b22274f295ff8cc6b52cf1121e2cff5010745c71 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 5 Jul 2024 18:18:10 +0200 Subject: [PATCH] Improve coverage output (#224) * Reformat coverage and move export to command * Add simple html coverage report * Fix coverage badge * Add basic documentation --- docs/docs/cli/coverage.md | 19 +++ mkdocs.yml | 1 + src/main/java/com/askimed/nf/test/App.java | 9 +- .../nf/test/commands/CoverageCommand.java | 96 +++++++++++++++ .../nf/test/commands/RunTestsCommand.java | 18 +-- .../nf/test/lang/dependencies/Coverage.java | 111 +++++++++++++++++- .../lang/dependencies/CoverageItemSorter.java | 9 ++ .../lang/dependencies/coverage-report.html | 48 ++++++++ 8 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 docs/docs/cli/coverage.md create mode 100644 src/main/java/com/askimed/nf/test/commands/CoverageCommand.java create mode 100644 src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java create mode 100644 src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html diff --git a/docs/docs/cli/coverage.md b/docs/docs/cli/coverage.md new file mode 100644 index 00000000..2d29f111 --- /dev/null +++ b/docs/docs/cli/coverage.md @@ -0,0 +1,19 @@ +# `coverage` command + +:octicons-tag-24: 0.9.0 + +## Usage + +``` +nf-test coverage +``` + +The `coverage` command prints information about the number of Nextflow files that are covered by a test. + +### Optional Arguments + +#### `--csv ` +Writes a coverage report in csv format. + +#### `--html ` +Writes a coverage report in html format. diff --git a/mkdocs.yml b/mkdocs.yml index 29829fd3..9667650f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ nav: - generate: docs/cli/generate.md - test: docs/cli/test.md - list: docs/cli/list.md + - coverage: docs/cli/coverage.md - clean: docs/cli/clean.md - Configuration: docs/configuration.md - Plugins: diff --git a/src/main/java/com/askimed/nf/test/App.java b/src/main/java/com/askimed/nf/test/App.java index ba7a7e26..d9db5d1e 100644 --- a/src/main/java/com/askimed/nf/test/App.java +++ b/src/main/java/com/askimed/nf/test/App.java @@ -1,12 +1,6 @@ package com.askimed.nf.test; -import com.askimed.nf.test.commands.CleanCommand; -import com.askimed.nf.test.commands.GenerateTestsCommand; -import com.askimed.nf.test.commands.InitCommand; -import com.askimed.nf.test.commands.ListTestsCommand; -import com.askimed.nf.test.commands.RunTestsCommand; -import com.askimed.nf.test.commands.UpdatePluginsCommand; -import com.askimed.nf.test.commands.VersionCommand; +import com.askimed.nf.test.commands.*; import ch.qos.logback.classic.Level; import picocli.CommandLine; @@ -35,6 +29,7 @@ public int run(String[] args) { commandLine.addSubcommand("clean", new CleanCommand()); commandLine.addSubcommand("init", new InitCommand()); commandLine.addSubcommand("test", new RunTestsCommand()); + commandLine.addSubcommand("coverage", new CoverageCommand()); commandLine.addSubcommand("list", new ListTestsCommand()); commandLine.addSubcommand("ls", new ListTestsCommand()); commandLine.addSubcommand("generate", new GenerateTestsCommand()); diff --git a/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java new file mode 100644 index 00000000..757962c4 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/commands/CoverageCommand.java @@ -0,0 +1,96 @@ +package com.askimed.nf.test.commands; + +import com.askimed.nf.test.config.Config; +import com.askimed.nf.test.lang.dependencies.Coverage; +import com.askimed.nf.test.lang.dependencies.DependencyResolver; +import com.askimed.nf.test.util.AnsiColors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Help.Visibility; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@Command(name = "coverage") +public class CoverageCommand extends AbstractCommand { + + private static final String SHARD_STRATEGY_ROUND_ROBIN = "round-robin"; + + @Option(names = { + "--csv" }, description = "Write coverage results in csv format", required = false, showDefaultValue = Visibility.ALWAYS) + private String csv = null; + + @Option(names = { + "--html" }, description = "Write coverage results in html format", required = false, showDefaultValue = Visibility.ALWAYS) + private String html = null; + + + @Option(names = { "--config", + "-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS) + private String configFilename = Config.FILENAME; + + private static Logger log = LoggerFactory.getLogger(CoverageCommand.class); + + @Override + public Integer execute() throws Exception { + + List scripts = new ArrayList(); + Config config = null; + + try { + + File defaultConfigFile = null; + boolean defaultWithTrace = true; + try { + File configFile = new File(configFilename); + if (configFile.exists()) { + log.info("Load config from file {}...", configFile.getAbsolutePath()); + config = Config.parse(configFile); + } else { + System.out.println(AnsiColors.yellow("Warning: This pipeline has no nf-test config file.")); + log.warn("No nf-test config file found."); + } + + } catch (Exception e) { + + System.out.println(AnsiColors.red("Error: Syntax errors in nf-test config file: " + e)); + log.error("Parsing config file failed", e); + return 2; + + } + + File baseDir = new File(new File("").getAbsolutePath()); + DependencyResolver resolver = new DependencyResolver(baseDir); + resolver.setFollowingDependencies(true); + + + if (config != null) { + resolver.buildGraph(config.getIgnore(), config.getTriggers()); + } else { + resolver.buildGraph(); + } + + Coverage coverage = new Coverage(resolver).getAll(); + if (csv != null) { + coverage.exportAsCsv(csv); + } else if (html != null) { + coverage.exportAsHtml(html); + } else { + coverage.printDetails(); + } + + return 0; + + } catch (Throwable e) { + + System.out.println(AnsiColors.red("Error: " + e));log.error("Running tests failed.", e); + return 1; + + } + + } + +} diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index ab8e8e2a..e43a7e08 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -188,7 +188,6 @@ public Integer execute() throws Exception { return 2; } - List ignorePatterns = new Vector(); File baseDir = new File(new File("").getAbsolutePath()); DependencyResolver resolver = new DependencyResolver(baseDir); resolver.setFollowingDependencies(followDependencies); @@ -231,10 +230,6 @@ public Integer execute() throws Exception { AnsiText.printBulletList(scripts); - if (coverage) { - new Coverage(resolver).getByFiles(testPaths).print(); - } - } else { if (config != null) { resolver.buildGraph(config.getIgnore(), config.getTriggers()); @@ -242,9 +237,6 @@ public Integer execute() throws Exception { resolver.buildGraph(); } scripts = resolver.findTestsByFiles(testPaths); - if (coverage) { - new Coverage(resolver).getAll().print(); - } } if (graph != null) { @@ -304,7 +296,15 @@ public Integer execute() throws Exception { System.out.println(AnsiColors.yellow("Dry run mode activated: tests are not executed, just listed.")); } - return engine.execute(); + int exitStatus = engine.execute(); + + if (coverage && findRelatedTests) { + new Coverage(resolver).getByFiles(testPaths).print(); + } else if (coverage) { + new Coverage(resolver).getAll().print(); + } + + return exitStatus; } catch (Throwable e) { diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java index 3d38cf62..cd85c6ea 100644 --- a/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/Coverage.java @@ -1,16 +1,31 @@ package com.askimed.nf.test.lang.dependencies; +import com.askimed.nf.test.commands.init.InitTemplates; +import com.askimed.nf.test.core.TestExecutionResult; +import com.askimed.nf.test.core.TestSuiteExecutionResult; +import com.askimed.nf.test.core.reports.CsvReportWriter; import com.askimed.nf.test.util.AnsiColors; +import com.askimed.nf.test.util.AnsiText; +import com.askimed.nf.test.util.FileUtil; +import com.opencsv.CSVWriter; +import groovy.lang.Writable; +import groovy.text.SimpleTemplateEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Paths; import java.text.DecimalFormat; -import java.util.List; -import java.util.Vector; +import java.text.DecimalFormatSymbols; +import java.util.*; public class Coverage { + private static final String HTML_TEMPLATE = "coverage-report.html"; + private int coveredItems = 0; private DependencyGraph graph; @@ -19,12 +34,15 @@ public class Coverage { private static Logger log = LoggerFactory.getLogger(Coverage.class); + private File baseDir = null; + public Coverage(DependencyGraph graph) { this.graph = graph; } public Coverage(DependencyResolver resolver) { this.graph = resolver.getGraph(); + baseDir = resolver.getBaseDir(); } public void add(File file, boolean covered) { @@ -57,6 +75,8 @@ public Coverage getAll(){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -87,6 +107,8 @@ public Coverage getByFiles(List files){ } + items.sort(new CoverageItemSorter()); + long time1 = System.currentTimeMillis(); log.info("Calculated coverage for {} files in {} sec", graph.size(), (time1 - time0) / 1000.0); @@ -95,14 +117,89 @@ public Coverage getByFiles(List files){ } public void print() { - DecimalFormat decimalFormat = new DecimalFormat("#.##"); + printLabel(); System.out.println(); - System.out.print("Coverage: " + getCoveredItems() + "/" + getItems().size()); - System.out.println(" (" + decimalFormat.format(getCoveredItems() / (float) getItems().size() * 100) + "%)"); + } + + public void printDetails() { + System.out.println(); + System.out.println("Files:"); for (Coverage.CoverageItem item : getItems()) { - System.out.println(" - " + (item.isCovered() ? AnsiColors.green(item.getFile().getAbsolutePath()) : AnsiColors.red(item.getFile().getAbsolutePath()))); + String label = getFileLabel(item.getFile()); + System.out.println(" \u2022 " + (item.isCovered() ? AnsiColors.green(label) : AnsiColors.red(label))); } System.out.println(); + printLabel(); + System.out.println(); + } + + public String getFileLabel(File file) { + String label = file.getAbsolutePath(); + if (baseDir != null) { + label = Paths.get(baseDir.getAbsolutePath()).relativize(file.toPath()).toString(); + } + return label; + } + + private void printLabel() { + float coverage = getCoveredItems() / (float) getItems().size(); + System.out.print(getColor("COVERAGE:", coverage) + " " + formatCoverage(coverage)); + System.out.println( " [" + getCoveredItems() + " of " + getItems().size() + " files]"); + } + + public float getCoverage() { + return getCoveredItems() / (float) getItems().size(); + } + + private String getColor(String label, float value) { + if (value < 0.5) { + return AnsiColors.red(label); + } else if (value < 0.9) { + return AnsiColors.yellow(label); + } else { + return AnsiColors.green(label); + } + } + + private String formatCoverage(float value) { + DecimalFormat decimalFormat = new DecimalFormat("#.##", DecimalFormatSymbols.getInstance(Locale.US)); + return decimalFormat.format(value * 100) + "%"; + } + + public void exportAsCsv(String filename) throws IOException { + String[] header = new String[]{ + "filename", + "covered", + "type" + }; + + CSVWriter writer = new CSVWriter(new FileWriter(new File(filename))); + writer.writeNext(header); + for (Coverage.CoverageItem item : getItems()) { + String[] line = new String[]{ + item.getFile().getAbsolutePath(), + item.isCovered() + "", + "unknown" + }; + + writer.writeNext(line); + } + + writer.close(); + System.out.println(); + printLabel(); + System.out.println(); + System.out.println("Wrote coverage report to file " + filename + "\n"); + + } + + public void exportAsHtml(String filename) throws IOException, ClassNotFoundException { + Map binding = new HashMap(); + binding.put("coverage", this); + URL templateUrl = Coverage.class.getResource(HTML_TEMPLATE); + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + Writable template = engine.createTemplate(templateUrl).make(binding); + FileUtil.write(new File(filename), template); } public static class CoverageItem { @@ -111,6 +208,8 @@ public static class CoverageItem { private boolean covered = false; + //TODO: add number of tests?? + public CoverageItem(File file, boolean covered) { this.file = file; this.covered = covered; diff --git a/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java new file mode 100644 index 00000000..4c7b1fd3 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/lang/dependencies/CoverageItemSorter.java @@ -0,0 +1,9 @@ +package com.askimed.nf.test.lang.dependencies; + +public class CoverageItemSorter implements java.util.Comparator { + + @Override + public int compare(Coverage.CoverageItem o1, Coverage.CoverageItem o2) { + return o1.getFile().getAbsolutePath().compareTo(o2.getFile().getAbsolutePath()); + } +} diff --git a/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html new file mode 100644 index 00000000..d0ad39b6 --- /dev/null +++ b/src/main/resources/com/askimed/nf/test/lang/dependencies/coverage-report.html @@ -0,0 +1,48 @@ + + + + + + + + + + +
+

Coverage Report

+

This report was generated by nf-test on <%= new Date() %>.

+

+ Coverage: <%= coverage.formatCoverage(coverage.getCoverage()) %> +

+ +
+
+
+ +
+ + + + + + + + + + <% coverage.items.each { item -> %> + + + + + <% } %> + +
FileCovered
<%= coverage.getFileLabel(item.getFile()) %><%= item.isCovered() %>
+
+ + \ No newline at end of file