From c3b61f86e50a35c3a7aefb7e85c043127996bb81 Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Tue, 3 Mar 2020 10:00:34 +0530 Subject: [PATCH] Configure TestNG to fail when all tests are skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #217 Currently TestNG doesn’t throw an exception which build tools can leverage to fail a build, when all the “@Test” methods get skipped and nothing was run. TestNG returns the proper status, but sometimes build tools may not be reading that status and take appropriate action. Introduced a new configuration through this PR using which a user can instruct TestNG itself to trigger an exception when no tests were executed. If one is using surefire plugin, then your surefire plugin configuration would look like below: failwheneverythingskipped true If you are using command line, then this feature can be turned on by passing -failwheneverythingskipped=true --- CHANGES.txt | 3 +- src/main/java/org/testng/CommandLineArgs.java | 7 ++++ src/main/java/org/testng/TestNG.java | 34 ++++++++++++++----- .../org/testng/internal/ExitCodeListener.java | 16 ++++++++- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 47dbcc4f33..338abbab1e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ Current -Fixed: GITHUB2255: Ensure test method parameters are visible in BeforeMethod config method (Krishnan Mahadevan) +Fixed:GITHUB-217: Configure TestNG to fail when all tests are skipped (Krishnan Mahadevan) +Fixed: GITHUB-2255: Ensure test method parameters are visible in BeforeMethod config method (Krishnan Mahadevan) Fixed: GITHUB-2251: NullPointerException at test with timeOut (Krishnan Mahadevan) Fixed: GITHUB-2249: Not abstract super-classes mess up test run order (Sergii Kim) Fixed: GITHUB-2195: NPE Using groups and @Before/@AfterMethod with alwaysRun and dependsOnMethods (Tomas & Julien Herr) diff --git a/src/main/java/org/testng/CommandLineArgs.java b/src/main/java/org/testng/CommandLineArgs.java index b99a9b0a11..d97e80a199 100644 --- a/src/main/java/org/testng/CommandLineArgs.java +++ b/src/main/java/org/testng/CommandLineArgs.java @@ -222,4 +222,11 @@ public class CommandLineArgs { description = "The dependency injector factory implementation that TestNG should use.") public String dependencyInjectorFactoryClass; + public static final String FAIL_IF_ALL_TESTS_SKIPPED = "-failwheneverythingskipped"; + + @Parameter( + names = FAIL_IF_ALL_TESTS_SKIPPED, + description = "Should TestNG fail execution if all tests were skipped and nothing was run.") + public Boolean failIfAllTestsSkipped = false; + } diff --git a/src/main/java/org/testng/TestNG.java b/src/main/java/org/testng/TestNG.java index 78a1ba5318..cbbb66ac15 100644 --- a/src/main/java/org/testng/TestNG.java +++ b/src/main/java/org/testng/TestNG.java @@ -100,6 +100,7 @@ * @see #usage() * @author Cedric Beust */ +@SuppressWarnings({"unused", "unchecked", "rawtypes"}) public class TestNG { /** This class' log4testng Logger. */ @@ -131,6 +132,7 @@ public class TestNG { private Boolean m_isJUnit = XmlSuite.DEFAULT_JUNIT; private Boolean m_isMixed = XmlSuite.DEFAULT_MIXED; protected boolean m_useDefaultListeners = true; + private boolean m_failIfAllTestsSkipped = false; private ITestRunnerFactory m_testRunnerFactory; @@ -156,7 +158,7 @@ public class TestNG { private int m_threadCount = -1; private XmlSuite.ParallelMode m_parallelMode = null; private XmlSuite.FailurePolicy m_configFailurePolicy; - private Class[] m_commandLineTestClasses; + private Class[] m_commandLineTestClasses; private String m_defaultSuiteName = DEFAULT_COMMAND_LINE_SUITE_NAME; private String m_defaultTestName = DEFAULT_COMMAND_LINE_TEST_NAME; @@ -216,6 +218,15 @@ private void init(boolean useDefaultListeners) { m_configuration = new Configuration(); } + /** + * @param failIfAllTestsSkipped - Whether TestNG should enable/disable failing when all the tests + * were skipped and nothing was run (Mostly when a test is powered by a data provider and when the + * data provider itself fails causing all tests to skip). + */ + public void toggleFailureIfAllTestsWereSkipped(boolean failIfAllTestsSkipped) { + this.m_failIfAllTestsSkipped = failIfAllTestsSkipped; + } + public int getStatus() { if (exitCodeListener.noTestsFound()) { return ExitCode.HAS_NO_TEST; @@ -254,7 +265,7 @@ public void setUseDefaultListeners(boolean useDefaultListeners) { /** * Sets a jar containing a testng.xml file. * - * @param jarPath + * @param jarPath - Path of the jar */ public void setTestJar(String jarPath) { m_jarPath = jarPath; @@ -501,7 +512,7 @@ private List createCommandLineSuitesForClasses(Class[] classes) { IAnnotationFinder finder = m_configuration.getAnnotationFinder(); for (int i = 0; i < classes.length; i++) { - Class c = classes[i]; + Class c = classes[i]; ITestAnnotation test = finder.findAnnotation(c, ITestAnnotation.class); String suiteName = getDefaultSuiteName(); String testName = getDefaultTestName(); @@ -576,7 +587,7 @@ public void setTestSuites(List suites) { /** * Specifies the XmlSuite objects to run. * - * @param suites + * @param suites - The list of {@link XmlSuite} objects. * @see org.testng.xml.XmlSuite */ public void setXmlSuites(List suites) { @@ -871,6 +882,9 @@ private void addReporter(Class r) { } private void initializeDefaultListeners() { + if (m_failIfAllTestsSkipped) { + this.exitCodeListener.failIfAllTestsSkipped(); + } addListener(this.exitCodeListener); if (m_useDefaultListeners) { addReporter(SuiteHTMLReporter.class); @@ -1365,7 +1379,7 @@ protected void configure(CommandLineArgs cla) { String testClasses = cla.testClass; if (null != testClasses) { String[] strClasses = testClasses.split(","); - List classes = Lists.newArrayList(); + List> classes = Lists.newArrayList(); for (String c : strClasses) { classes.add(ClassHelper.fileToClass(c)); } @@ -1392,6 +1406,8 @@ protected void configure(CommandLineArgs cla) { setJUnit(cla.junit); setMixed(cla.mixed); setSkipFailedInvocationCounts(cla.skipFailedInvocationCounts); + toggleFailureIfAllTestsWereSkipped(cla.failIfAllTestsSkipped); + if (cla.parallelMode != null) { setParallel(cla.parallelMode); } @@ -1451,8 +1467,8 @@ protected void configure(CommandLineArgs cla) { setTestRunnerFactoryClass(ClassHelper.fileToClass(cla.testRunnerFactory)); } - if (cla.reporter != null) { - ReporterConfig reporterConfig = ReporterConfig.deserialize(cla.reporter); + ReporterConfig reporterConfig = ReporterConfig.deserialize(cla.reporter); + if (reporterConfig != null) { addReporter(reporterConfig); } @@ -1546,6 +1562,8 @@ public void configure(Map cmdLineArgs) { result.mixed = (Boolean) cmdLineArgs.get(CommandLineArgs.MIXED); result.skipFailedInvocationCounts = (Boolean) cmdLineArgs.get(CommandLineArgs.SKIP_FAILED_INVOCATION_COUNTS); + result.failIfAllTestsSkipped = Boolean.parseBoolean( + cmdLineArgs.getOrDefault(CommandLineArgs.FAIL_IF_ALL_TESTS_SKIPPED, Boolean.FALSE).toString()); String parallelMode = (String) cmdLineArgs.get(CommandLineArgs.PARALLEL); if (parallelMode != null) { result.parallelMode = XmlSuite.ParallelMode.getValidParallel(parallelMode); @@ -1637,7 +1655,7 @@ private void addReporter(ReporterConfig reporterConfig) { /** * Specify if this run should be made in JUnit mode * - * @param isJUnit + * @param isJUnit - Specify if this run should be made in JUnit mode */ public void setJUnit(Boolean isJUnit) { m_isJUnit = isJUnit; diff --git a/src/main/java/org/testng/internal/ExitCodeListener.java b/src/main/java/org/testng/internal/ExitCodeListener.java index 2ad1eaf79c..6e82b9ab30 100644 --- a/src/main/java/org/testng/internal/ExitCodeListener.java +++ b/src/main/java/org/testng/internal/ExitCodeListener.java @@ -1,18 +1,25 @@ package org.testng.internal; +import org.testng.IExecutionListener; import org.testng.IReporter; import org.testng.ISuite; import org.testng.ISuiteResult; import org.testng.ITestContext; import org.testng.ITestListener; import org.testng.ITestResult; +import org.testng.TestNGException; import org.testng.xml.XmlSuite; import java.util.List; -public class ExitCodeListener implements ITestListener, IReporter { +public class ExitCodeListener implements ITestListener, IReporter, IExecutionListener { private boolean hasTests = false; private final ExitCode status = new ExitCode(); + private boolean failIfAllTestsSkipped = false; + + public void failIfAllTestsSkipped() { + this.failIfAllTestsSkipped = true; + } public ExitCode getStatus() { return status; @@ -63,4 +70,11 @@ public void onStart(ITestContext context) {} @Override public void onFinish(ITestContext context) {} + + @Override + public void onExecutionFinish() { + if (failIfAllTestsSkipped) { + throw new TestNGException("All tests were skipped. Nothing was run."); + } + } }