Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix #7 - ignore invalid XML files #8

Merged
merged 1 commit into from
Aug 31, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
fix #7 - ignore invalid XML files
  • Loading branch information
pihme committed Aug 31, 2020
commit e65a99cd42b2519e05a19741c7295ec665dd5e46
Original file line number Diff line number Diff line change
@@ -20,7 +20,8 @@

/**
* Based on
* {@code org.apache.maven.plugins.surefire.report.SurefireReportParser} (licensed under http://www.apache.org/licenses/LICENSE-2.0).
* {@code org.apache.maven.plugins.surefire.report.SurefireReportParser}
* (licensed under http://www.apache.org/licenses/LICENSE-2.0).
*
* This implementation uses the extended data objects to capture more
* information on flaky tests
@@ -53,15 +54,16 @@ public Map<File, List<ExtendedReportTestSuite>> parseXMLReportFiles() throws Par
final Map<File, List<ExtendedReportTestSuite>> result = new HashMap<>();

final ExtendedTestSuiteXMLParser parser = new ExtendedTestSuiteXMLParser(logger);
for (File aXmlReportFileList : xmlReportFiles) {
for (File xmlReportFile : xmlReportFiles) {
try {
result.put(aXmlReportFileList, parser.parse(aXmlReportFileList.getAbsolutePath()));
result.put(xmlReportFile, parser.parse(xmlReportFile.getAbsolutePath()));
} catch (ParserConfigurationException e) {
throw new ParsingException("Error setting up parser for JUnit XML report", e);
} catch (SAXException e) {
throw new ParsingException("Error parsing JUnit XML report " + aXmlReportFileList, e);
logger.info("Skipping " + xmlReportFile.getName() + " because of parsing exception:"
+ e.getLocalizedMessage());
} catch (IOException e) {
throw new ParsingException("Error reading JUnit XML report " + aXmlReportFileList, e);
throw new ParsingException("Error reading JUnit XML report " + xmlReportFile, e);
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package io.zeebe.flakytestextractor;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

public class ExtendedSurefireReportParserTest {

private static String[] RESSOURCES = new String[] {
// bad file that should be skipped
"invalid-xml-examples/TEST-io.zeebe.broker.it.clustering.BrokerLeaderChangeTest-it-testrun.xml",
// good files that should be parsed
"surefire-reports/TEST-com.github.pihme.jenkinstestbed.module1.FlakyTest.xml",
"surefire-reports/TEST-com.github.pihme.jenkinstestbed.module1.PassingTest.xml" };

@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();

@Before
public void setUpFiles() throws IOException {
TestUtil.copyClassPatHResourcesToFolder(RESSOURCES, tempFolder.getRoot());
}

@Test
public void shouldShouldNotThrowParsingExceptionForInvalidXML() {
// given
ExtendedSurefireReportParser sut = new ExtendedSurefireReportParser(
Collections.singletonList(tempFolder.getRoot()), Locale.US, new TestLogger());

// when
try {
sut.parseXMLReportFiles();
} catch (ParsingException e) {
Assertions.fail("Parser threw parsing exception");
}
}

@Test
public void shouldIgnoreInvalidXMLButParseTheRest() throws ParsingException {
// given
ExtendedSurefireReportParser sut = new ExtendedSurefireReportParser(
Collections.singletonList(tempFolder.getRoot()), Locale.US, new TestLogger());

// when
Map<File, List<ExtendedReportTestSuite>> parsedFiles = sut.parseXMLReportFiles();

// then
assertThat(parsedFiles).describedAs("Successfully parsed files").hasSize(2);
}

}
Original file line number Diff line number Diff line change
@@ -13,10 +13,6 @@
import org.junit.Test;
import org.xml.sax.SAXException;

import io.zeebe.flakytestextractor.ExtendedReportTestCase;
import io.zeebe.flakytestextractor.ExtendedReportTestSuite;
import io.zeebe.flakytestextractor.ExtendedTestSuiteXMLParser;

public class ExtendedTestSuiteXMLParserTest {
private static final ClassLoader CLASS_LOADER = Thread.currentThread().getContextClassLoader();

Original file line number Diff line number Diff line change
@@ -10,30 +10,19 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.logging.Logger;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.io.FileUtils;
import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.xml.sax.SAXException;

import io.zeebe.flakytestextractor.ExtendedReportTestCase;
import io.zeebe.flakytestextractor.ExtendedReportTestSuite;
import io.zeebe.flakytestextractor.ExtendedTestSuiteXMLParser;
import io.zeebe.flakytestextractor.FlakyTestExtractorPlugin;

public class FlakyTestExtractorPluginTest {

private static final Logger LOGGER = Logger.getLogger("com.github.pihme.flakytestextractor");

private final ExtendedTestSuiteXMLParser PARSER = new ExtendedTestSuiteXMLParser(new TestLogger());

private static String[] RESSOURCES = new String[] {
@@ -48,14 +37,7 @@ public class FlakyTestExtractorPluginTest {

@Before
public void setUpFiles() throws IOException {
for (String resource : RESSOURCES) {
URL url = ClassLoader.getSystemResource(resource);
String urlString = url.toExternalForm();
String targetName = urlString.substring(urlString.lastIndexOf("/"));
File destination = new File(tempFolder.getRoot(), targetName);
FileUtils.copyURLToFile(url, destination);
LOGGER.info("Copied " + url + " to " + destination.getAbsolutePath());
}
TestUtil.copyClassPatHResourcesToFolder(RESSOURCES, tempFolder.getRoot());
}

@Test
50 changes: 50 additions & 0 deletions src/test/java/io/zeebe/flakytestextractor/InvalidXMLTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package io.zeebe.flakytestextractor;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.io.File;
import java.io.IOException;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
* This tests covers cases where the XML test reports are invalid. The desired
* outcome is that the plugin can skip over invalid XML gracefully
*/
public class InvalidXMLTest {

private static String[] RESSOURCES = new String[] {
// bad file that should be skipped
"invalid-xml-examples/TEST-io.zeebe.broker.it.clustering.BrokerLeaderChangeTest-it-testrun.xml",
// good files that should be parsed
"surefire-reports/TEST-com.github.pihme.jenkinstestbed.module1.FlakyTest.xml",
"surefire-reports/TEST-com.github.pihme.jenkinstestbed.module1.PassingTest.xml" };

@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();

@Before
public void setUpFiles() throws IOException {
TestUtil.copyClassPatHResourcesToFolder(RESSOURCES, tempFolder.getRoot());
}

@Test
public void testShouldIgnoreInvalidXMLAndParseValidXML() throws Exception {
// given
FlakyTestExtractorPlugin sut = new FlakyTestExtractorPlugin();
sut.reportDir = tempFolder.getRoot();

// when
assertThatThrownBy(() -> sut.execute()).hasMessage("Flaky tests encountered");

// then
File[] createdFiles = tempFolder.getRoot().listFiles(file -> file.getName().endsWith("-FLAKY.xml"));

assertThat(createdFiles).hasSize(1);
}

}
2 changes: 1 addition & 1 deletion src/test/java/io/zeebe/flakytestextractor/TestLogger.java
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@

public class TestLogger implements Log {

private static final Logger logger = Logger.getLogger("com.github.pihme.flakytestextractor");
private static final Logger logger = Logger.getLogger("io.zeebe.flakytestextractor");

@Override
public boolean isDebugEnabled() {
25 changes: 25 additions & 0 deletions src/test/java/io/zeebe/flakytestextractor/TestUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package io.zeebe.flakytestextractor;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.logging.Logger;

import org.apache.commons.io.FileUtils;

public class TestUtil {

private static final Logger LOGGER = Logger.getLogger("io.zeebe.flakytestextracto");

public static void copyClassPatHResourcesToFolder(String[] ressourcesInClassPath, File destinationFolder) throws IOException {
for (String resource : ressourcesInClassPath) {
URL url = ClassLoader.getSystemResource(resource);
String urlString = url.toExternalForm();
String targetName = urlString.substring(urlString.lastIndexOf("/"));
File destination = new File(destinationFolder, targetName);
FileUtils.copyURLToFile(url, destination);
LOGGER.info("Copied " + url + " to " + destination.getAbsolutePath());
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd" version="3.0" name="io.zeebe.broker.it.clustering.BrokerLeaderChangeTest(it-testrun)" time="33.74" tests="1" errors="0" skipped="0" failures="0">
<properties>
<property name="awt.toolkit" value="sun.awt.X11.XToolkit"/>
<property name="java.specification.version" value="11"/>
<property name="sun.cpu.isalist" value=""/>
<property name="sun.jnu.encoding" value="UTF-8"/>
<property name="java.class.version" value="55.0"/>
</properties>
<testcase name="shouldBeAbleToBecomeLeaderAgain" classname="io.zeebe.broker.it.clustering.BrokerLeaderChangeTest(it-testrun)" time="28.364"/>
<testcase name="shouldChangeLeaderAfterLeaderDies" classname="io.zeebe.broker.it.clustering.BrokerLeaderChangeTest(it-testrun)" time="27.225"/>
<testcase name="shouldBecomeFollowerAfterRestartLeaderChange" classname="io.zeebe.broker.it.clustering.BrokerLeaderChangeTest(it-testrun)" time="33.738">
<flakyError message="Lambda expression in io.zeebe.broker.it.clustering.ClusteringRule: expected the predicate to return &lt;true&gt; but it returned &lt;false&gt; for input of &lt;[BrokerInfoImpl{nodeId=0, host=&apos;0.0.0.0&apos;, port=28812, version=0.25.0-SNAPSHOT, partitions=[PartitionInfoImpl{partitionId=1, role=FOLLOWER}]}, BrokerInfoImpl{nodeId=2, host=&apos;0.0.0.0&apos;, port=28822, version=0.25.0-SNAPSHOT, partitions=[PartitionInfoImpl{partitionId=1, role=FOLLOWER}]}]&gt; within 10 seconds." type="org.awaitility.core.ConditionTimeoutException">
<stackTrace><![CDATA[org.awaitility.core.ConditionTimeoutException: Lambda expression in io.zeebe.broker.it.clustering.ClusteringRule: expected the predicate to return <true> but it returned <false> for input of <[BrokerInfoImpl{nodeId=0, host='0.0.0.0', port=28812, version=0.25.0-SNAPSHOT, partitions=[PartitionInfoImpl{partitionId=1, role=FOLLOWER}]}, BrokerInfoImpl{nodeId=2, host='0.0.0.0', port=28822, version=0.25.0-SNAPSHOT, partitions=[PartitionInfoImpl{partitionId=1, role=FOLLOWER}]}]> within 10 seconds.
at org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:165)
at org.awaitility.core.AbstractHamcrestCondition.await(AbstractHamcrestCondition.java:86)
at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:895)
at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:601)
at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:618)
at io.zeebe.broker.it.clustering.ClusteringRule.waitForTopology(ClusteringRule.java:619)
at io.zeebe.broker.it.clustering.ClusteringRule.waitForNewLeaderOfPartitions(ClusteringRule.java:600)
at io.zeebe.broker.it.clustering.ClusteringRule.stopBrokerAndAwaitNewLeader(ClusteringRule.java:574)
at io.zeebe.broker.it.clustering.BrokerLeaderChangeTest.shouldBecomeFollowerAfterRestartLeaderChange(BrokerLeaderChangeTest.java:53)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:288)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:282)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.util.concurrent.TimeoutException
at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:204)
at org.awaitility.core.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:101)
at org.awaitility.core.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:81)
at org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:101)
... 25 more
]]></stackTrace>
<system-out><![CDATA[