diff --git a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java index e6503507d..fe2082e6f 100644 --- a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java +++ b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java @@ -45,6 +45,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -310,24 +311,26 @@ && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485 if (errorMessage == null) { try { TestExecutionResult result = testPluginAgainst(actualCoreCoordinates, plugin, mconfig, pomData, pluginsToCheck, pluginGroupIds, pcth, config.getOverridenPlugins()); - if (result.getTestDetails().getFailed().isEmpty()) { + if (result.getTestDetails().isSuccess()) { status = TestStatus.SUCCESS; } else { status = TestStatus.TEST_FAILURES; } warningMessages.addAll(result.pomWarningMessages); - testDetails.addAll(config.isStoreAll() ? result.getTestDetails().getAll() : result.getTestDetails().getFailed()); + testDetails.addAll(config.isStoreAll() ? result.getTestDetails().getAll() : result.getTestDetails().hasFailures() ? result.getTestDetails().getFailed() : Collections.emptySet()); } catch (PomExecutionException e) { if(!e.succeededPluginArtifactIds.contains("maven-compiler-plugin")){ status = TestStatus.COMPILATION_ERROR; - } else if (!e.getTestDetails().getFailed().isEmpty()) { - status = TestStatus.TEST_FAILURES; + } else if (!e.getTestDetails().hasBeenExecuted()) { // testing was not able to start properly (i.e: invalid exclusion list file format) + status = TestStatus.INTERNAL_ERROR; + } else if (e.getTestDetails().hasFailures()) { + status = TestStatus.TEST_FAILURES; } else { // Can this really happen ??? status = TestStatus.SUCCESS; } errorMessage = e.getErrorMessage(); warningMessages.addAll(e.getPomWarningMessages()); - testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().getFailed()); + testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().hasFailures() ? e.getTestDetails().getFailed() : Collections.emptySet()); } catch (Error e){ // Rethrow the error ... something is wrong ! throw e; @@ -566,7 +569,7 @@ private TestExecutionResult testPluginAgainst(MavenCoordinates coreCoordinates, } catch (PomExecutionException e){ e.getPomWarningMessages().addAll(pomData.getWarningMessages()); if (ranCompile) { - // So the status is considered to be TEST_FAILURES not COMPILATION_ERROR: + // So the status cannot be considered COMPILATION_ERROR e.succeededPluginArtifactIds.add("maven-compiler-plugin"); } throw e; diff --git a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java index 2b520a422..b8a8eb2c3 100644 --- a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java +++ b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java @@ -16,8 +16,6 @@ public class ExecutedTestNamesDetails { public ExecutedTestNamesDetails() { this.tests = new HashMap<>(); - this.tests.put(FAILED, new TreeSet()); - this.tests.put(EXECUTED, new TreeSet()); } public void addFailedTest(String test) { @@ -29,8 +27,13 @@ public void addExecutedTest(String test) { } public Set getAll() { - Set result = new TreeSet<>(this.tests.get(EXECUTED)); - result.addAll(this.tests.get(FAILED)); + Set result = new TreeSet<>(); + if (this.tests.containsKey(EXECUTED)) { + result.addAll(this.tests.get(EXECUTED)); + } + if (this.tests.containsKey(FAILED)) { + result.addAll(this.tests.get(FAILED)); + } return Collections.unmodifiableSet(result); } @@ -43,13 +46,25 @@ public Set getExecuted() { } private Set get(String key) { - return Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key))); + return this.tests.containsKey(key) ? Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key))) : null; } private void add(String key, String test) { + if (this.tests.get(key) == null) { + this.tests.put(key, new TreeSet()); + } this.tests.get(key).add(test); } - - + public boolean hasBeenExecuted() { + return getExecuted() != null || getFailed() != null; + } + + public boolean isSuccess() { + return getExecuted() != null && getFailed() == null; + } + + public boolean hasFailures() { + return getFailed() != null && !getFailed().isEmpty(); + } } diff --git a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java index 83e5be5cb..6c126cf0e 100644 --- a/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java +++ b/plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java @@ -101,18 +101,20 @@ public ExecutedTestNamesDetails solve(Set types, Set executedTes System.out.println("[INFO] "); System.out.println("[INFO] Results:"); + printDetails(testNames.getExecuted(), "Executed"); + printDetails(testNames.getFailed(), "Failed"); + return testNames; + } + + private void printDetails(Set tests, String type) { System.out.println("[INFO] "); - System.out.println(String.format("[INFO] Executed: %s", testNames.getExecuted().size())); - for (String testName : testNames.getExecuted()) { - System.out.println(String.format("[INFO] - %s", testName)); - } - System.out.println("[INFO] "); - System.out.println(String.format("[INFO] Failed: %s", testNames.getFailed().size())); - for (String testName : testNames.getFailed()) { - System.out.println(String.format("[INFO] - %s", testName)); + int size = tests != null ? tests.size() : 0; + System.out.println(String.format("[INFO] %s: %s", type, size)); + if (size != 0) { + for (String testName : tests) { + System.out.println(String.format("[INFO] - %s", testName)); + } } - - return testNames; } private List getReportsDirectoryPaths(Set types, File baseDirectory) throws ExecutedTestNamesSolverException { diff --git a/plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java b/plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java index 0f7f43352..2f7f986ef 100644 --- a/plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java +++ b/plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java @@ -213,6 +213,30 @@ public void testWithCasCProperties() throws Throwable { PluginCompatTester tester = new PluginCompatTester(config); tester.testPlugins(); } + + @Test + public void testWithInvalidExclusionList() throws Throwable { + File exclusionList = new ClassPathResource("bad-surefire-exclusion-list").getFile(); + PluginCompatTesterConfig config = getConfig(ImmutableList.of("active-directory")); + Map mavenProperties = new HashMap<>(); + mavenProperties.put("surefire.excludesFile", exclusionList.getAbsolutePath()); + config.setMavenProperties(mavenProperties); + + PluginCompatTester tester = new PluginCompatTester(config); + PluginCompatReport report = tester.testPlugins(); + assertNotNull(report); + Map> pluginCompatTests = report.getPluginCompatTests(); + assertNotNull(pluginCompatTests); + for (Entry> entry : pluginCompatTests.entrySet()) { + assertEquals("active-directory", entry.getKey().pluginName); + List results = entry.getValue(); + assertEquals(1, results.size()); + PluginCompatResult result = results.get(0); + assertNotNull(result); + assertNotNull(result.status); + assertEquals(TestStatus.INTERNAL_ERROR, result.status); + } + } private PluginCompatTesterConfig getConfig(List includedPlugins) throws IOException { PluginCompatTesterConfig config = new PluginCompatTesterConfig(testFolder.getRoot(), diff --git a/plugins-compat-tester/src/test/resources/bad-surefire-exclusion-list b/plugins-compat-tester/src/test/resources/bad-surefire-exclusion-list new file mode 100644 index 000000000..310004eaf --- /dev/null +++ b/plugins-compat-tester/src/test/resources/bad-surefire-exclusion-list @@ -0,0 +1 @@ +io.jenkins.plugins.FooBar#method \ No newline at end of file