Skip to content

Commit

Permalink
Refactor running of data driven tests (#4466)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Evangelink
Evangelink authored Dec 29, 2024
1 parent c743f67 commit 56cacdd
Showing 1 changed file with 125 additions and 110 deletions.
235 changes: 125 additions & 110 deletions src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,43 +161,54 @@ internal UnitTestResult[] RunTestMethod()
DebugEx.Assert(_testMethodInfo.TestMethod != null, "Test method should not be null.");

List<TestResult> results = [];
bool isDataDriven = false;
var parentStopwatch = Stopwatch.StartNew();

if (_testMethodInfo.TestMethodOptions.Executor != null)
if (_testMethodInfo.TestMethodOptions.Executor == null)
{
if (_test.DataType == DynamicDataType.ITestDataSource)
{
object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
TestResult[] testResults = ExecuteTestWithDataSource(null, data);
results.AddRange(testResults);
}
else if (ExecuteDataSourceBasedTests(results))
{
isDataDriven = true;
}
else
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
_testMethodInfo.TestClassName,
_testMethodInfo.TestMethodName);

TestResult emptyResult = new()
{
_testContext.SetDisplayName(_test.DisplayName);
TestResult[] testResults = ExecuteTest(_testMethodInfo);
Outcome = UTF.UnitTestOutcome.Unknown,
TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult),
};
_testContext.SetOutcome(emptyResult.Outcome);

foreach (TestResult testResult in testResults)
{
if (StringEx.IsNullOrWhiteSpace(testResult.DisplayName))
{
testResult.DisplayName = _test.DisplayName;
}
}
results.Add(emptyResult);
return results.ToUnitTestResults();
}

results.AddRange(testResults);
}
bool isDataDriven = false;
var parentStopwatch = Stopwatch.StartNew();
if (_test.DataType == DynamicDataType.ITestDataSource)
{
object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
TestResult[] testResults = ExecuteTestWithDataSource(null, data);
results.AddRange(testResults);
}
else if (TryExecuteDataSourceBasedTests(results))
{
isDataDriven = true;
}
else if (TryExecuteFoldedDataDrivenTests(results))
{
isDataDriven = true;
}
else
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
_testMethodInfo.TestClassName,
_testMethodInfo.TestMethodName);
_testContext.SetDisplayName(_test.DisplayName);
TestResult[] testResults = ExecuteTest(_testMethodInfo);

foreach (TestResult testResult in testResults)
{
if (StringEx.IsNullOrWhiteSpace(testResult.DisplayName))
{
testResult.DisplayName = _test.DisplayName;
}
}

results.AddRange(testResults);
}

// Get aggregate outcome.
Expand All @@ -210,6 +221,7 @@ internal UnitTestResult[] RunTestMethod()
// In legacy scenario
#pragma warning disable CS0618 // Type or member is obsolete
if (_test.TestIdGenerationStrategy == TestIdGenerationStrategy.Legacy)
#pragma warning restore CS0618 // Type or member is obsolete
{
parentStopwatch.Stop();
var parentResult = new TestResult
Expand All @@ -221,7 +233,6 @@ internal UnitTestResult[] RunTestMethod()

results = UpdateResultsWithParentInfo(results, parentResult);
}
#pragma warning restore CS0618 // Type or member is obsolete
else
{
results = UpdateResultsWithParentInfo(results);
Expand All @@ -243,108 +254,112 @@ internal UnitTestResult[] RunTestMethod()
return results.ToUnitTestResults();
}

private bool ExecuteDataSourceBasedTests(List<TestResult> results)
private bool TryExecuteDataSourceBasedTests(List<TestResult> results)
{
bool isDataDriven = false;

DataSourceAttribute[] dataSourceAttribute = _testMethodInfo.GetAttributes<DataSourceAttribute>(false);
if (dataSourceAttribute is { Length: 1 })
{
isDataDriven = true;
Stopwatch watch = new();
watch.Start();
ExecuteTestFromDataSourceAttribute(results);
return true;
}

try
return false;
}

private bool TryExecuteFoldedDataDrivenTests(List<TestResult> results)
{
IEnumerable<UTF.ITestDataSource>? testDataSources = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>();
if (testDataSources?.Any() != true)
{
return false;
}

foreach (UTF.ITestDataSource testDataSource in testDataSources)
{
IEnumerable<object?[]>? dataSource;

// This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests.
// Any change made here should be reflected in AssemblyEnumerator.ProcessTestDataSourceTests as well.
dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);

if (!dataSource.Any())
{
IEnumerable<object>? dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(_testMethodInfo, _testContext);
if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
{
throw testDataSource.GetExceptionForEmptyDataSource(_testMethodInfo.MethodInfo);
}

var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
};
results.Add(inconclusiveResult);
continue;
}

if (dataRows == null)
foreach (object?[] data in dataSource)
{
try
{
var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
Duration = watch.Elapsed,
};
results.Add(inconclusiveResult);
TestResult[] testResults = ExecuteTestWithDataSource(testDataSource, data);

results.AddRange(testResults);
}
else
finally
{
try
{
int rowIndex = 0;

foreach (object dataRow in dataRows)
{
TestResult[] testResults = ExecuteTestWithDataRow(dataRow, rowIndex++);
results.AddRange(testResults);
}
}
finally
{
_testContext.SetDataConnection(null);
_testContext.SetDataRow(null);
}
_testMethodInfo.SetArguments(null);
}
}
catch (Exception ex)
}

return true;
}

private void ExecuteTestFromDataSourceAttribute(List<TestResult> results)
{
Stopwatch watch = new();
watch.Start();

try
{
IEnumerable<object>? dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(_testMethodInfo, _testContext);
if (dataRows == null)
{
var failedResult = new TestResult
var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Error,
TestFailureException = ex,
Outcome = UTF.UnitTestOutcome.Inconclusive,
Duration = watch.Elapsed,
};
results.Add(failedResult);
results.Add(inconclusiveResult);
return;
}
}
else
{
IEnumerable<UTF.ITestDataSource>? testDataSources = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>();

if (testDataSources != null)
try
{
foreach (UTF.ITestDataSource testDataSource in testDataSources)
int rowIndex = 0;

foreach (object dataRow in dataRows)
{
isDataDriven = true;
IEnumerable<object?[]>? dataSource;

// This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests.
// Any change made here should be reflected in AssemblyEnumerator.ProcessTestDataSourceTests as well.
dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);

if (!dataSource.Any())
{
if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
{
throw testDataSource.GetExceptionForEmptyDataSource(_testMethodInfo.MethodInfo);
}

var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
};
results.Add(inconclusiveResult);
continue;
}

foreach (object?[] data in dataSource)
{
try
{
TestResult[] testResults = ExecuteTestWithDataSource(testDataSource, data);

results.AddRange(testResults);
}
finally
{
_testMethodInfo.SetArguments(null);
}
}
TestResult[] testResults = ExecuteTestWithDataRow(dataRow, rowIndex++);
results.AddRange(testResults);
}
}
finally
{
_testContext.SetDataConnection(null);
_testContext.SetDataRow(null);
}
}
catch (Exception ex)
{
var failedResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Error,
TestFailureException = ex,
Duration = watch.Elapsed,
};
results.Add(failedResult);
}

return isDataDriven;
}

private TestResult[] ExecuteTestWithDataSource(UTF.ITestDataSource? testDataSource, object?[]? data)
Expand Down

0 comments on commit 56cacdd

Please sign in to comment.