Configure TestNG to fail when all tests are skipped

Closes testng-team#217

Currently TestNG doesn’t throw an exception which
build tools can leverage to fail a build, when all
the “@test” methods get skipped and nothing was run.
TestNG returns the proper status, but sometimes 
build tools may not be reading that status and take
appropriate action.

Introduced a new configuration through this PR using
which a user can instruct TestNG itself to trigger
an exception when no tests were executed.

If one is using surefire plugin, then your surefire
plugin configuration would look like below:


<configuration>
  <properties>
    <property>
      <name>failwheneverythingskipped</name>
      <value>true</value>
    </property>
  </properties>
</configuration>

If you are using command line, then this feature
can be turned on by passing 
-failwheneverythingskipped=true

CHANGES.txt

@@ -1,5 +1,6 @@
 Current
-Fixed: GITHUB2255: Ensure test method parameters are visible in BeforeMethod config method (Krishnan Mahadevan)
+Fixed:GITHUB-217: Configure TestNG to fail when all tests are skipped (Krishnan Mahadevan)
+Fixed: GITHUB-2255: Ensure test method parameters are visible in BeforeMethod config method (Krishnan Mahadevan)
 Fixed: GITHUB-2251: NullPointerException at test with timeOut (Krishnan Mahadevan)
 Fixed: GITHUB-2249: Not abstract super-classes mess up test run order (Sergii Kim)
 Fixed: GITHUB-2195: NPE Using groups and @Before/@AfterMethod with alwaysRun and dependsOnMethods (Tomas & Julien Herr)

src/main/java/org/testng/CommandLineArgs.java

@@ -222,4 +222,11 @@ public class CommandLineArgs {
       description = "The dependency injector factory implementation that TestNG should use.")
   public String dependencyInjectorFactoryClass;
 
+  public static final String FAIL_IF_ALL_TESTS_SKIPPED = "-failwheneverythingskipped";
+
+  @Parameter(
+      names = FAIL_IF_ALL_TESTS_SKIPPED,
+      description = "Should TestNG fail execution if all tests were skipped and nothing was run.")
+  public Boolean failIfAllTestsSkipped = false;
+
 }

src/main/java/org/testng/TestNG.java

@@ -100,6 +100,7 @@
  * @see #usage()
  * @author <a href = "mailto:cedric&#64;beust.com">Cedric Beust</a>
  */
+@SuppressWarnings({"unused", "unchecked", "rawtypes"})
 public class TestNG {
 
   /** This class' log4testng Logger. */
@@ -131,6 +132,7 @@ public class TestNG {
   private Boolean m_isJUnit = XmlSuite.DEFAULT_JUNIT;
   private Boolean m_isMixed = XmlSuite.DEFAULT_MIXED;
   protected boolean m_useDefaultListeners = true;
+  private boolean m_failIfAllTestsSkipped = false;
 
   private ITestRunnerFactory m_testRunnerFactory;
 
@@ -156,7 +158,7 @@ public class TestNG {
   private int m_threadCount = -1;
   private XmlSuite.ParallelMode m_parallelMode = null;
   private XmlSuite.FailurePolicy m_configFailurePolicy;
-  private Class[] m_commandLineTestClasses;
+  private Class<?>[] m_commandLineTestClasses;
 
   private String m_defaultSuiteName = DEFAULT_COMMAND_LINE_SUITE_NAME;
   private String m_defaultTestName = DEFAULT_COMMAND_LINE_TEST_NAME;
@@ -216,6 +218,15 @@ private void init(boolean useDefaultListeners) {
     m_configuration = new Configuration();
   }
 
+  /**
+   * @param failIfAllTestsSkipped - Whether TestNG should enable/disable failing when all the tests
+   * were skipped and nothing was run (Mostly when a test is powered by a data provider and when the
+   * data provider itself fails causing all tests to skip).
+   */
+  public void toggleFailureIfAllTestsWereSkipped(boolean failIfAllTestsSkipped) {
+    this.m_failIfAllTestsSkipped = failIfAllTestsSkipped;
+  }
+
   public int getStatus() {
     if (exitCodeListener.noTestsFound()) {
       return ExitCode.HAS_NO_TEST;
@@ -254,7 +265,7 @@ public void setUseDefaultListeners(boolean useDefaultListeners) {
   /**
    * Sets a jar containing a testng.xml file.
    *
-   * @param jarPath
+   * @param jarPath - Path of the jar
    */
   public void setTestJar(String jarPath) {
     m_jarPath = jarPath;
@@ -501,7 +512,7 @@ private List<XmlSuite> createCommandLineSuitesForClasses(Class[] classes) {
     IAnnotationFinder finder = m_configuration.getAnnotationFinder();
 
     for (int i = 0; i < classes.length; i++) {
-      Class c = classes[i];
+      Class<?> c = classes[i];
       ITestAnnotation test = finder.findAnnotation(c, ITestAnnotation.class);
       String suiteName = getDefaultSuiteName();
       String testName = getDefaultTestName();
@@ -576,7 +587,7 @@ public void setTestSuites(List<String> suites) {
   /**
    * Specifies the XmlSuite objects to run.
    *
-   * @param suites
+   * @param suites - The list of {@link XmlSuite} objects.
    * @see org.testng.xml.XmlSuite
    */
   public void setXmlSuites(List<XmlSuite> suites) {
@@ -871,6 +882,9 @@ private void addReporter(Class<? extends IReporter> r) {
   }
 
   private void initializeDefaultListeners() {
+    if (m_failIfAllTestsSkipped) {
+      this.exitCodeListener.failIfAllTestsSkipped();
+    }
     addListener(this.exitCodeListener);
     if (m_useDefaultListeners) {
       addReporter(SuiteHTMLReporter.class);
@@ -1365,7 +1379,7 @@ protected void configure(CommandLineArgs cla) {
     String testClasses = cla.testClass;
     if (null != testClasses) {
       String[] strClasses = testClasses.split(",");
-      List<Class> classes = Lists.newArrayList();
+      List<Class<?>> classes = Lists.newArrayList();
       for (String c : strClasses) {
         classes.add(ClassHelper.fileToClass(c));
       }
@@ -1392,6 +1406,8 @@ protected void configure(CommandLineArgs cla) {
     setJUnit(cla.junit);
     setMixed(cla.mixed);
     setSkipFailedInvocationCounts(cla.skipFailedInvocationCounts);
+    toggleFailureIfAllTestsWereSkipped(cla.failIfAllTestsSkipped);
+
     if (cla.parallelMode != null) {
       setParallel(cla.parallelMode);
     }
@@ -1451,8 +1467,8 @@ protected void configure(CommandLineArgs cla) {
       setTestRunnerFactoryClass(ClassHelper.fileToClass(cla.testRunnerFactory));
     }
 
-    if (cla.reporter != null) {
-      ReporterConfig reporterConfig = ReporterConfig.deserialize(cla.reporter);
+    ReporterConfig reporterConfig = ReporterConfig.deserialize(cla.reporter);
+    if (reporterConfig != null) {
       addReporter(reporterConfig);
     }
 
@@ -1546,6 +1562,8 @@ public void configure(Map cmdLineArgs) {
     result.mixed = (Boolean) cmdLineArgs.get(CommandLineArgs.MIXED);
     result.skipFailedInvocationCounts =
         (Boolean) cmdLineArgs.get(CommandLineArgs.SKIP_FAILED_INVOCATION_COUNTS);
+    result.failIfAllTestsSkipped = Boolean.parseBoolean(
+        cmdLineArgs.getOrDefault(CommandLineArgs.FAIL_IF_ALL_TESTS_SKIPPED, Boolean.FALSE).toString());
     String parallelMode = (String) cmdLineArgs.get(CommandLineArgs.PARALLEL);
     if (parallelMode != null) {
       result.parallelMode = XmlSuite.ParallelMode.getValidParallel(parallelMode);
@@ -1637,7 +1655,7 @@ private void addReporter(ReporterConfig reporterConfig) {
   /**
    * Specify if this run should be made in JUnit mode
    *
-   * @param isJUnit
+   * @param isJUnit - Specify if this run should be made in JUnit mode
    */
   public void setJUnit(Boolean isJUnit) {
     m_isJUnit = isJUnit;

src/main/java/org/testng/internal/ExitCodeListener.java

@@ -1,18 +1,25 @@
 package org.testng.internal;
 
+import org.testng.IExecutionListener;
 import org.testng.IReporter;
 import org.testng.ISuite;
 import org.testng.ISuiteResult;
 import org.testng.ITestContext;
 import org.testng.ITestListener;
 import org.testng.ITestResult;
+import org.testng.TestNGException;
 import org.testng.xml.XmlSuite;
 
 import java.util.List;
 
-public class ExitCodeListener implements ITestListener, IReporter {
+public class ExitCodeListener implements ITestListener, IReporter, IExecutionListener {
   private boolean hasTests = false;
   private final ExitCode status = new ExitCode();
+  private boolean failIfAllTestsSkipped = false;
+
+  public void failIfAllTestsSkipped() {
+    this.failIfAllTestsSkipped = true;
+  }
 
   public ExitCode getStatus() {
     return status;
@@ -63,4 +70,11 @@ public void onStart(ITestContext context) {}
 
   @Override
   public void onFinish(ITestContext context) {}
+
+  @Override
+  public void onExecutionFinish() {
+    if (failIfAllTestsSkipped) {
+      throw new TestNGException("All tests were skipped. Nothing was run.");
+    }
+  }
 }