Skip to content

Commit

Permalink
Assert test run status in CI Vis instrumentation tests (#8316)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nikita-tkachenko-datadog
nikita-tkachenko-datadog authored Jan 31, 2025
1 parent f7e62de commit b24d153
Show file tree
Hide file tree
Showing 22 changed files with 533 additions and 988 deletions.
44 changes: 28 additions & 16 deletions ...ava-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,21 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
def runner = new JUnitCore()

def "test #testcaseName"() {
runFeatures(features)
runFeatures(features, success)

assertSpansData(testcaseName)

where:
testcaseName | features
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
"test-multiple-features-${version()}" | [
testcaseName | success | features
"test-succeed" | true | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
"test-scenario-outline-${version()}" | true | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
"test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
"test-multiple-features-${version()}" | false | [
"org/example/cucumber/calculator/basic_arithmetic.feature",
"org/example/cucumber/calculator/basic_arithmetic_failed.feature"
]
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"]
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
"test-name-with-brackets" | true | ["org/example/cucumber/calculator/name_with_brackets.feature"]
"test-empty-name-${version()}" | true | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
}

def "test ITR #testcaseName"() {
Expand All @@ -56,17 +56,17 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
givenFlakyRetryEnabled(true)
givenFlakyTests(retriedTests)

runFeatures(features)
runFeatures(features, success)

assertSpansData(testcaseName)

where:
testcaseName | features | retriedTests
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
testcaseName | success | features | retriedTests
"test-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-retry-failure" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
"test-retry-scenario-outline-${version()}" | false | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null)
]
}
Expand All @@ -93,16 +93,28 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
return CucumberTracingListener.FRAMEWORK_VERSION < "7" ? CucumberTracingListener.FRAMEWORK_VERSION : "latest"
}

private void runFeatures(List<String> classpathFeatures) {
private void runFeatures(List<String> classpathFeatures, boolean expectSuccess = true) {
System.setProperty(Constants.GLUE_PROPERTY_NAME, "org.example.cucumber.calculator")
System.setProperty(Constants.FILTER_TAGS_PROPERTY_NAME, "not @Disabled")
System.setProperty(Constants.FEATURES_PROPERTY_NAME, classpathFeatures.stream()
.map(f -> "classpath:" + f).
collect(Collectors.joining(",")))

TestEventsHandlerHolder.start()
runner.run(TestSucceedCucumber)
TestEventsHandlerHolder.stop()
try {
def result = runner.run(TestSucceedCucumber)
if (expectSuccess) {
if (result.getFailureCount() > 0) {
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
}
} else {
if (result.getFailureCount() == 0) {
throw new AssertionError("Expected a failed execution, got no failures")
}
}
} finally {
TestEventsHandlerHolder.stop()
}
}

@Override
Expand Down
40 changes: 24 additions & 16 deletions dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,40 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
def runner = new JUnitCore()

def "test #testcaseName"() {
runTests(tests)
runTests(tests, success)

assertSpansData(testcaseName)

where:
testcaseName | tests
"test-succeed-before-after" | [TestSucceedBeforeAfter]
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass]
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam]
"test-failed-before-class" | [TestFailedBeforeClass]
"test-failed-after-class" | [TestFailedAfterClass]
"test-failed-before" | [TestFailedBefore]
"test-failed-after" | [TestFailedAfter]
"test-failed-before-param" | [TestFailedBeforeParam]
"test-failed-after-param" | [TestFailedAfterParam]
testcaseName | success | tests
"test-succeed-before-after" | true | [TestSucceedBeforeAfter]
"test-succeed-before-class-after-class" | true | [TestSucceedBeforeClassAfterClass]
"test-succeed-before-param-after-param" | true | [TestSucceedBeforeParamAfterParam]
"test-failed-before-class" | false | [TestFailedBeforeClass]
"test-failed-after-class" | false | [TestFailedAfterClass]
"test-failed-before" | false | [TestFailedBefore]
"test-failed-after" | false | [TestFailedAfter]
"test-failed-before-param" | false | [TestFailedBeforeParam]
"test-failed-after-param" | false | [TestFailedAfterParam]
}

private void runTests(Collection<Class<?>> tests) {
private void runTests(Collection<Class<?>> tests, boolean expectSuccess = true) {
TestEventsHandlerHolder.start()
try {
Class[] array = tests.toArray(new Class[0])
runner.run(array)
} catch (Throwable ignored) {
// Ignored
def result = runner.run(array)
if (expectSuccess) {
if (result.getFailureCount() > 0) {
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
}
} else {
if (result.getFailureCount() == 0) {
throw new AssertionError("Expected a failed execution, got no failures")
}
}
} finally {
TestEventsHandlerHolder.stop()
}
TestEventsHandlerHolder.stop()
}

@Override
Expand Down
28 changes: 18 additions & 10 deletions dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
givenFlakyRetryEnabled(true)
givenFlakyTests(retriedTests)

runTests(tests)
runTests(tests, success)

assertSpansData(testcaseName)

where:
testcaseName | tests | retriedTests
"test-failed" | [TestFailedMUnit] | []
"test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
testcaseName | success | tests | retriedTests
"test-failed" | false | [TestFailedMUnit] | []
"test-retry-failed" | false | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
"test-failed-then-succeed" | true | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
}

def "test early flakiness detection #testcaseName"() {
Expand Down Expand Up @@ -79,15 +79,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
"test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
}

private void runTests(Collection<Class<?>> tests) {
private void runTests(Collection<Class<?>> tests, boolean expectSuccess = true) {
TestEventsHandlerHolder.start()
try {
Class[] array = tests.toArray(new Class[0])
runner.run(array)
} catch (Throwable ignored) {
// Ignored
def result = runner.run(array)
if (expectSuccess) {
if (result.getFailureCount() > 0) {
throw new AssertionError("Expected successful execution, got following failures: " + result.getFailures())
}
} else {
if (result.getFailureCount() == 0) {
throw new AssertionError("Expected a failed execution, got no failures")
}
}
} finally {
TestEventsHandlerHolder.stop()
}
TestEventsHandlerHolder.stop()
}

String version() {
Expand Down
Loading

0 comments on commit b24d153

Please sign in to comment.