Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor running of data driven tests #4466

Merged
merged 2 commits into from
Dec 29, 2024
Merged

Refactor running of data driven tests #4466

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5848304
Refactor running of data driven tests
Evangelink Dec 29, 2024
ffefa02
More refactoring
Evangelink Dec 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 125 additions & 110 deletions src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,43 +161,54 @@ internal UnitTestResult[] RunTestMethod()
DebugEx.Assert(_testMethodInfo.TestMethod != null, "Test method should not be null.");

List<TestResult> results = [];
bool isDataDriven = false;
var parentStopwatch = Stopwatch.StartNew();

if (_testMethodInfo.TestMethodOptions.Executor != null)
if (_testMethodInfo.TestMethodOptions.Executor == null)
{
if (_test.DataType == DynamicDataType.ITestDataSource)
{
object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
TestResult[] testResults = ExecuteTestWithDataSource(null, data);
results.AddRange(testResults);
}
else if (ExecuteDataSourceBasedTests(results))
{
isDataDriven = true;
}
else
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
_testMethodInfo.TestClassName,
_testMethodInfo.TestMethodName);

TestResult emptyResult = new()
{
_testContext.SetDisplayName(_test.DisplayName);
TestResult[] testResults = ExecuteTest(_testMethodInfo);
Outcome = UTF.UnitTestOutcome.Unknown,
TestFailureException = new TestFailedException(UnitTestOutcome.Error, Resource.UTA_NoTestResult),
};
_testContext.SetOutcome(emptyResult.Outcome);

foreach (TestResult testResult in testResults)
{
if (StringEx.IsNullOrWhiteSpace(testResult.DisplayName))
{
testResult.DisplayName = _test.DisplayName;
}
}
results.Add(emptyResult);
return results.ToUnitTestResults();
}

results.AddRange(testResults);
}
bool isDataDriven = false;
var parentStopwatch = Stopwatch.StartNew();
if (_test.DataType == DynamicDataType.ITestDataSource)
{
object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
TestResult[] testResults = ExecuteTestWithDataSource(null, data);
results.AddRange(testResults);
}
else if (TryExecuteDataSourceBasedTests(results))
{
isDataDriven = true;
}
else if (TryExecuteFoldedDataDrivenTests(results))
{
isDataDriven = true;
}
else
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(
"Not able to get executor for method {0}.{1}",
_testMethodInfo.TestClassName,
_testMethodInfo.TestMethodName);
_testContext.SetDisplayName(_test.DisplayName);
TestResult[] testResults = ExecuteTest(_testMethodInfo);

foreach (TestResult testResult in testResults)
{
if (StringEx.IsNullOrWhiteSpace(testResult.DisplayName))
{
testResult.DisplayName = _test.DisplayName;
}
}

results.AddRange(testResults);
}

// Get aggregate outcome.
Expand All @@ -210,6 +221,7 @@ internal UnitTestResult[] RunTestMethod()
// In legacy scenario
#pragma warning disable CS0618 // Type or member is obsolete
if (_test.TestIdGenerationStrategy == TestIdGenerationStrategy.Legacy)
#pragma warning restore CS0618 // Type or member is obsolete
{
parentStopwatch.Stop();
var parentResult = new TestResult
Expand All @@ -221,7 +233,6 @@ internal UnitTestResult[] RunTestMethod()

results = UpdateResultsWithParentInfo(results, parentResult);
}
#pragma warning restore CS0618 // Type or member is obsolete
else
{
results = UpdateResultsWithParentInfo(results);
Expand All @@ -243,108 +254,112 @@ internal UnitTestResult[] RunTestMethod()
return results.ToUnitTestResults();
}

private bool ExecuteDataSourceBasedTests(List<TestResult> results)
private bool TryExecuteDataSourceBasedTests(List<TestResult> results)
{
bool isDataDriven = false;

DataSourceAttribute[] dataSourceAttribute = _testMethodInfo.GetAttributes<DataSourceAttribute>(false);
if (dataSourceAttribute is { Length: 1 })
{
isDataDriven = true;
Stopwatch watch = new();
watch.Start();
ExecuteTestFromDataSourceAttribute(results);
return true;
}

try
return false;
}

private bool TryExecuteFoldedDataDrivenTests(List<TestResult> results)
{
IEnumerable<UTF.ITestDataSource>? testDataSources = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>();
if (testDataSources?.Any() != true)
{
return false;
}

foreach (UTF.ITestDataSource testDataSource in testDataSources)
{
IEnumerable<object?[]>? dataSource;

// This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests.
// Any change made here should be reflected in AssemblyEnumerator.ProcessTestDataSourceTests as well.
dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);

if (!dataSource.Any())
{
IEnumerable<object>? dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(_testMethodInfo, _testContext);
if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
{
throw testDataSource.GetExceptionForEmptyDataSource(_testMethodInfo.MethodInfo);
}

var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
};
results.Add(inconclusiveResult);
continue;
}

if (dataRows == null)
foreach (object?[] data in dataSource)
{
try
{
var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
Duration = watch.Elapsed,
};
results.Add(inconclusiveResult);
TestResult[] testResults = ExecuteTestWithDataSource(testDataSource, data);

results.AddRange(testResults);
}
else
finally
{
try
{
int rowIndex = 0;

foreach (object dataRow in dataRows)
{
TestResult[] testResults = ExecuteTestWithDataRow(dataRow, rowIndex++);
results.AddRange(testResults);
}
}
finally
{
_testContext.SetDataConnection(null);
_testContext.SetDataRow(null);
}
_testMethodInfo.SetArguments(null);
}
}
catch (Exception ex)
}

return true;
}

private void ExecuteTestFromDataSourceAttribute(List<TestResult> results)
{
Stopwatch watch = new();
watch.Start();

try
{
IEnumerable<object>? dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(_testMethodInfo, _testContext);
if (dataRows == null)
{
var failedResult = new TestResult
var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Error,
TestFailureException = ex,
Outcome = UTF.UnitTestOutcome.Inconclusive,
Duration = watch.Elapsed,
};
results.Add(failedResult);
results.Add(inconclusiveResult);
return;
}
}
else
{
IEnumerable<UTF.ITestDataSource>? testDataSources = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>();

if (testDataSources != null)
try
{
foreach (UTF.ITestDataSource testDataSource in testDataSources)
int rowIndex = 0;

foreach (object dataRow in dataRows)
{
isDataDriven = true;
IEnumerable<object?[]>? dataSource;

// This code is to execute tests. To discover the tests code is in AssemblyEnumerator.ProcessTestDataSourceTests.
// Any change made here should be reflected in AssemblyEnumerator.ProcessTestDataSourceTests as well.
dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);

if (!dataSource.Any())
{
if (!MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
{
throw testDataSource.GetExceptionForEmptyDataSource(_testMethodInfo.MethodInfo);
}

var inconclusiveResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Inconclusive,
};
results.Add(inconclusiveResult);
continue;
}

foreach (object?[] data in dataSource)
{
try
{
TestResult[] testResults = ExecuteTestWithDataSource(testDataSource, data);

results.AddRange(testResults);
}
finally
{
_testMethodInfo.SetArguments(null);
}
}
TestResult[] testResults = ExecuteTestWithDataRow(dataRow, rowIndex++);
results.AddRange(testResults);
}
}
finally
{
_testContext.SetDataConnection(null);
_testContext.SetDataRow(null);
}
}
catch (Exception ex)
{
var failedResult = new TestResult
{
Outcome = UTF.UnitTestOutcome.Error,
TestFailureException = ex,
Duration = watch.Elapsed,
};
results.Add(failedResult);
}

return isDataDriven;
}

private TestResult[] ExecuteTestWithDataSource(UTF.ITestDataSource? testDataSource, object?[]? data)
Expand Down