Skip to content

Commit

Permalink
Include all data driven tests for reporting skips
Browse files Browse the repository at this point in the history
Closes #2674

We now have a new configuration parameter named
“-includeAllDataDrivenTestsWhenSkipping” which 
when used would cause a data driven test to report
All its iterations as skipped when there is a failure
in its upstream dependencies.
  • Loading branch information
krmahadevan committed Nov 23, 2021
1 parent 4c28cd6 commit db17f3c
Show file tree
Hide file tree
Showing 9 changed files with 149 additions and 7 deletions.
1 change: 1 addition & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Current
Fixed: GITHUB-2674: Run onTestSkipped for each value from data provider (Krishnan Mahadevan)
Fixed: GITHUB-2672: Log real stacktrace when test times out. (cdalexndr)
Fixed: GITHUB-2669: A failed retry with ITestContext will lose the ITestContext. (Nan Liang)
Fixed: GITHUB-2643: assertEquals(Set,Set) now ignores ordering as it did before. (Elis Edlund)
Expand Down
9 changes: 9 additions & 0 deletions testng-core/src/main/java/org/testng/CommandLineArgs.java
Original file line number Diff line number Diff line change
Expand Up @@ -242,4 +242,13 @@ public class CommandLineArgs {
description =
"Comma separated fully qualified class names of listeners that should be skipped from being wired in via Service Loaders.")
public Boolean overrideIncludedMethods = false;

public static final String INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING =
"-includeAllDataDrivenTestsWhenSkipping";

@Parameter(
names = INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING,
description =
"Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.")
public Boolean includeAllDataDrivenTestsWhenSkipping = false;
}
13 changes: 13 additions & 0 deletions testng-core/src/main/java/org/testng/TestNG.java
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,14 @@ public void addMethodSelector(XmlMethodSelector selector) {
m_selectors.add(selector);
}

public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) {
this.m_configuration.setReportAllDataDrivenTestsAsSkipped(reportAllDataDrivenTestsAsSkipped);
}

public boolean getReportAllDataDrivenTestsAsSkipped() {
return this.m_configuration.getReportAllDataDrivenTestsAsSkipped();
}

/**
* Set the suites file names to be run by this TestNG object. This method tries to load and parse
* the specified TestNG suite xml files. If a file is missing, it is ignored.
Expand Down Expand Up @@ -1404,6 +1412,7 @@ public static TestNG privateMain(String[] argv, ITestListener listener) {
* @param cla The command line parameters
*/
protected void configure(CommandLineArgs cla) {
setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping);
if (cla.verbose != null) {
setVerbose(cla.verbose);
}
Expand Down Expand Up @@ -1613,6 +1622,10 @@ public void configure(Map cmdLineArgs) {
result.xmlPathInJar = (String) cmdLineArgs.get(CommandLineArgs.XML_PATH_IN_JAR);
result.junit = (Boolean) cmdLineArgs.get(CommandLineArgs.JUNIT);
result.mixed = (Boolean) cmdLineArgs.get(CommandLineArgs.MIXED);
Object tmpValue = cmdLineArgs.get(CommandLineArgs.INCLUDE_ALL_DATA_DRIVEN_TESTS_WHEN_SKIPPING);
if (tmpValue != null) {
result.includeAllDataDrivenTestsWhenSkipping = Boolean.parseBoolean(tmpValue.toString());
}
result.skipFailedInvocationCounts =
(Boolean) cmdLineArgs.get(CommandLineArgs.SKIP_FAILED_INVOCATION_COUNTS);
result.failIfAllTestsSkipped =
Expand Down
12 changes: 12 additions & 0 deletions testng-core/src/main/java/org/testng/internal/Configuration.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public class Configuration implements IConfiguration {
private IInjectorFactory injectorFactory = new GuiceBackedInjectorFactory();
private boolean overrideIncludedMethods = false;

private boolean includeAllDataDrivenTestsWhenSkipping;

public Configuration() {
init(new JDK15AnnotationFinder(new DefaultAnnotationTransformer()));
}
Expand Down Expand Up @@ -144,4 +146,14 @@ public boolean getOverrideIncludedMethods() {
public void setOverrideIncludedMethods(boolean overrideIncludedMethods) {
this.overrideIncludedMethods = overrideIncludedMethods;
}

@Override
public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) {
this.includeAllDataDrivenTestsWhenSkipping = reportAllDataDrivenTestsAsSkipped;
}

@Override
public boolean getReportAllDataDrivenTestsAsSkipped() {
return this.includeAllDataDrivenTestsWhenSkipping;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,10 @@ default boolean addExecutionListenerIfAbsent(IExecutionListener l) {
boolean getOverrideIncludedMethods();

void setOverrideIncludedMethods(boolean overrideIncludedMethods);

default void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTestsAsSkipped) {}

default boolean getReportAllDataDrivenTestsAsSkipped() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
Expand All @@ -16,6 +17,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.testng.DataProviderHolder;
import org.testng.DataProviderInvocationException;
Expand All @@ -37,6 +39,7 @@
import org.testng.SuiteRunner;
import org.testng.TestException;
import org.testng.TestNGException;
import org.testng.collections.CollectionUtils;
import org.testng.collections.Lists;
import org.testng.collections.Maps;
import org.testng.collections.Sets;
Expand Down Expand Up @@ -104,12 +107,48 @@ public List<ITestResult> invokeTestMethods(
//
// Not okToProceed. Test is being skipped
//
ITestResult result =
registerSkippedTestResult(
testMethod, System.currentTimeMillis(), new Throwable(okToProceed));
m_notifier.addSkippedTest(testMethod, result);
InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), result);
invokeListenersForSkippedTestResult(result, invokedMethod);
List<ITestResult> results = new ArrayList<>();
Consumer<ITestResult> resultProcessor =
result -> {
m_notifier.addSkippedTest(testMethod, result);
InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), result);
invokeListenersForSkippedTestResult(result, invokedMethod);
};
boolean reportAllDataDrivenTestsAsSkipped =
m_configuration.getReportAllDataDrivenTestsAsSkipped();
if (reportAllDataDrivenTestsAsSkipped && testMethod.isDataDriven()) {
ParameterHandler handler =
new ParameterHandler(
m_configuration.getObjectFactory(),
annotationFinder(),
buildDataProviderHolder(),
1);

ParameterBag bag =
handler.createParameters(
testMethod, Maps.newHashMap(), Maps.newHashMap(), context, instance);
Iterator<Object[]> allParamValues = Objects.requireNonNull(bag.parameterHolder).parameters;
Iterable<Object[]> allParameterValues = CollectionUtils.asIterable(allParamValues);
for (Object[] next : allParameterValues) {
if (next == null) {
continue;
}
Method m = testMethod.getConstructorOrMethod().getMethod();
Object[] parameterValues = Parameters.injectParameters(next, m, context);
ITestResult result =
registerSkippedTestResult(
testMethod, System.currentTimeMillis(), new Throwable(okToProceed));
result.setParameters(parameterValues);
resultProcessor.accept(result);
results.add(result);
}
} else {
ITestResult result =
registerSkippedTestResult(
testMethod, System.currentTimeMillis(), new Throwable(okToProceed));
resultProcessor.accept(result);
results.add(result);
}
testMethod.incrementCurrentInvocationCount();
GroupConfigMethodArguments args =
new Builder()
Expand All @@ -119,7 +158,7 @@ public List<ITestResult> invokeTestMethods(
.withParameters(parameters)
.build();
this.invoker.invokeAfterGroupsConfigurations(args);
return Collections.singletonList(result);
return Collections.unmodifiableList(results);
}

// For invocationCount > 1 and threadPoolSize > 1 run this method in its own pool thread.
Expand Down
28 changes: 28 additions & 0 deletions testng-core/src/test/java/test/skip/ReasonForSkipTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@

import java.util.Arrays;
import java.util.Map;
import org.testng.CommandLineArgs;
import org.testng.ITestResult;
import org.testng.TestNG;
import org.testng.annotations.Test;
import org.testng.collections.Maps;
import org.testng.xml.XmlSuite;
import test.InvokedMethodNameListener;
import test.SimpleBaseTest;
import test.skip.github1967.TestClassSample;
import test.skip.issue2674.ConfigAwareTestNG;

public class ReasonForSkipTest extends SimpleBaseTest {

Expand Down Expand Up @@ -103,6 +106,31 @@ public void testEnsureTestStatusIsSetProperlyForSkippedTests() {
assertThat(actual).containsAllEntriesOf(expected);
}

@Test(description = "GITHUB-2674")
public void ensureUpstreamFailuresTriggerSkipsForAllDataProviderValues() {
TestNG testng = create(test.skip.issue2674.TestClassSample.class);
testng.setReportAllDataDrivenTestsAsSkipped(true);
InvokedMethodNameListener listener = new InvokedMethodNameListener();
testng.addListener(listener);
testng.run();
assertThat(listener.getSkippedMethodNames())
.containsExactly("test2(iPhone,13)", "test2(iPhone-Pro,12)");
}

@Test(description = "GITHUB-2674")
public void ensureUpstreamFailuresTriggerSkipsForAllDataProviderValuesViaCmdLineArgs() {
CommandLineArgs cli = new CommandLineArgs();
cli.includeAllDataDrivenTestsWhenSkipping = true;
ConfigAwareTestNG testng = new ConfigAwareTestNG();
testng.setTestClasses(new Class<?>[] {test.skip.issue2674.TestClassSample.class});
testng.configure(cli);
InvokedMethodNameListener listener = new InvokedMethodNameListener();
testng.addListener(listener);
testng.run();
assertThat(listener.getSkippedMethodNames())
.containsExactly("test2(iPhone,13)", "test2(iPhone-Pro,12)");
}

private static void runTest(Map<String, String> expected, Class<?>... clazz) {
TestNG testng = create(clazz);
ReasonReporter reporter = new ReasonReporter();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package test.skip.issue2674;

import org.testng.CommandLineArgs;
import org.testng.TestNG;

public class ConfigAwareTestNG extends TestNG {

@Override
public void configure(CommandLineArgs cla) {
super.configure(cla);
}
}
22 changes: 22 additions & 0 deletions testng-core/src/test/java/test/skip/issue2674/TestClassSample.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package test.skip.issue2674;

import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class TestClassSample {
@Test
void test1() {
Assert.fail();
}

@Test(
dataProvider = "items",
dependsOnMethods = {"test1"})
void test2(String model, int variant) {}

@DataProvider(name = "items")
Object[][] items() {
return new Object[][] {{"iPhone", 13}, {"iPhone-Pro", 12}};
}
}

0 comments on commit db17f3c

Please sign in to comment.