Bugfix on dealing with non-executed test phase due collateral issues

plugins-compat-tester/src/main/java/org/jenkins/tools/test/PluginCompatTester.java

@@ -45,6 +45,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -310,7 +311,7 @@ && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485
                     if (errorMessage == null) {
                     try {
                         TestExecutionResult result = testPluginAgainst(actualCoreCoordinates, plugin, mconfig, pomData, pluginsToCheck, pluginGroupIds, pcth, config.getOverridenPlugins());
-                        if (result.getTestDetails().getFailed().isEmpty()) {
+                        if (result.getTestDetails().isSuccess()) {
                             status = TestStatus.SUCCESS;
                         } else {
                             status = TestStatus.TEST_FAILURES;
@@ -320,14 +321,16 @@ && new VersionNumber(coreCoordinates.version).compareTo(new VersionNumber("1.485
                     } catch (PomExecutionException e) {
                         if(!e.succeededPluginArtifactIds.contains("maven-compiler-plugin")){
                             status = TestStatus.COMPILATION_ERROR;
-                        } else if (!e.getTestDetails().getFailed().isEmpty()) {
-                            status = TestStatus.TEST_FAILURES;
+                        } else if (!e.getTestDetails().hasBeenExecuted()) { // testing was not able to start properly (i.e: invalid exclusion list file format)
+                            status = TestStatus.INTERNAL_ERROR;
+                        } else if (!e.getTestDetails().hasFailures()) { 
+                            status = TestStatus.TEST_FAILURES;                            
                         } else { // Can this really happen ???
                             status = TestStatus.SUCCESS;
                         }
                         errorMessage = e.getErrorMessage();
                         warningMessages.addAll(e.getPomWarningMessages());
-                        testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().getFailed());
+                        testDetails.addAll(config.isStoreAll() ? e.getTestDetails().getAll() : e.getTestDetails().hasFailures() ? e.getTestDetails().getFailed() : Collections.emptySet());
                     } catch (Error e){
                         // Rethrow the error ... something is wrong !
                         throw e;
@@ -566,7 +569,7 @@ private TestExecutionResult testPluginAgainst(MavenCoordinates coreCoordinates,
         } catch (PomExecutionException e){
             e.getPomWarningMessages().addAll(pomData.getWarningMessages());
             if (ranCompile) {
-                // So the status is considered to be TEST_FAILURES not COMPILATION_ERROR:
+                // So the status cannot be considered COMPILATION_ERROR
                 e.succeededPluginArtifactIds.add("maven-compiler-plugin");
             }
             throw e;

plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesDetails.java

@@ -16,8 +16,6 @@ public class ExecutedTestNamesDetails {
 
     public ExecutedTestNamesDetails() {
         this.tests = new HashMap<>();
-        this.tests.put(FAILED, new TreeSet<String>());
-        this.tests.put(EXECUTED, new TreeSet<String>());
     }
 
     public void addFailedTest(String test) {
@@ -29,8 +27,13 @@ public void addExecutedTest(String test) {
     }
 
     public Set<String> getAll() {
-        Set<String> result = new TreeSet<>(this.tests.get(EXECUTED));
-        result.addAll(this.tests.get(FAILED));
+        Set<String> result = new TreeSet<>();
+        if (this.tests.containsKey(EXECUTED)) {
+            result.addAll(this.tests.get(EXECUTED));
+        }
+        if (this.tests.containsKey(FAILED)) {
+            result.addAll(this.tests.get(FAILED));
+        }
         return Collections.unmodifiableSet(result);
     }
 
@@ -43,13 +46,25 @@ public Set<String> getExecuted() {
     }
 
     private Set<String> get(String key) {
-        return Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key)));
+        return this.tests.containsKey(key) ? Collections.unmodifiableSet(new TreeSet<>(this.tests.get(key))) : null;
     }
 
     private void add(String key, String test) {
+        if (this.tests.get(key) == null) {
+            this.tests.put(key, new TreeSet<String>());
+        }
         this.tests.get(key).add(test);
     }
-
-
 
+    public boolean hasBeenExecuted() {
+        return getExecuted() != null || getFailed() != null;
+    }
+
+    public boolean isSuccess() {
+        return getExecuted() != null && getFailed() == null;
+    }
+
+    public boolean hasFailures() {
+        return getFailed() != null && !getFailed().isEmpty();
+    }
 }

plugins-compat-tester/src/main/java/org/jenkins/tools/test/util/ExecutedTestNamesSolver.java

@@ -101,18 +101,20 @@ public ExecutedTestNamesDetails solve(Set<String> types, Set<String> executedTes
 
         System.out.println("[INFO] ");
         System.out.println("[INFO] Results:");
+        printDetails(testNames.getExecuted(), "Executed");
+        printDetails(testNames.getFailed(), "Failed");
+        return testNames;
+    }
+
+    private void printDetails(Set<String> tests, String type) {
         System.out.println("[INFO] ");
-        System.out.println(String.format("[INFO] Executed: %s", testNames.getExecuted().size()));
-        for (String testName : testNames.getExecuted()) {
-            System.out.println(String.format("[INFO] - %s", testName));
-        }
-        System.out.println("[INFO] ");
-        System.out.println(String.format("[INFO] Failed: %s", testNames.getFailed().size()));
-        for (String testName : testNames.getFailed()) {
-            System.out.println(String.format("[INFO] - %s", testName));
+        int size = tests != null ? tests.size() : 0;
+        System.out.println(String.format("[INFO] %s: %s", type, size));
+        if (size != 0) {
+            for (String testName : tests) {
+                System.out.println(String.format("[INFO] - %s", testName));
+            }
         }
-
-        return testNames;
     }
 
     private List<String> getReportsDirectoryPaths(Set<String> types, File baseDirectory) throws ExecutedTestNamesSolverException {

plugins-compat-tester/src/test/java/org/jenkins/tools/test/PluginCompatTesterTest.java

@@ -213,6 +213,30 @@ public void testWithCasCProperties() throws Throwable {
         PluginCompatTester tester = new PluginCompatTester(config);
         tester.testPlugins();
     }
+
+    @Test
+    public void testWithInvalidExclusionList() throws Throwable {
+        File exclusionList = new ClassPathResource("bad-surefire-exclusion-list").getFile();
+        PluginCompatTesterConfig config = getConfig(ImmutableList.of("active-directory"));
+        Map<String, String> mavenProperties = new HashMap<>();
+        mavenProperties.put("surefire.excludesFile", exclusionList.getAbsolutePath());
+        config.setMavenProperties(mavenProperties);
+
+        PluginCompatTester tester = new PluginCompatTester(config);
+        PluginCompatReport report = tester.testPlugins();
+        assertNotNull(report);
+        Map<PluginInfos, List<PluginCompatResult>> pluginCompatTests = report.getPluginCompatTests();
+        assertNotNull(pluginCompatTests);
+        for (Entry<PluginInfos, List<PluginCompatResult>> entry : pluginCompatTests.entrySet()) {
+            assertEquals("active-directory", entry.getKey().pluginName);
+            List<PluginCompatResult> results = entry.getValue();
+            assertEquals(1, results.size());
+            PluginCompatResult result = results.get(0);
+            assertNotNull(result);
+            assertNotNull(result.status);
+            assertEquals(TestStatus.INTERNAL_ERROR, result.status);
+        }
+    }
 
     private PluginCompatTesterConfig getConfig(List<String> includedPlugins) throws IOException {
         PluginCompatTesterConfig config = new PluginCompatTesterConfig(testFolder.getRoot(),

plugins-compat-tester/src/test/resources/bad-surefire-exclusion-list

@@ -0,0 +1 @@
+io.jenkins.plugins.FooBar#method