From ef7ff071099d185a5f66b55966866fcc964e287c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 11 Jan 2022 21:25:46 -0800 Subject: [PATCH] Capture and record output from tests from Console.Out into the test results (#63606) * Capture and record output from tests from Console.Out into the test results. * Simplify construction. * Fix nullability * Fix constructor visibility --- .../Common/XUnitWrapperGenerator/ITestInfo.cs | 11 +++--- .../XUnitWrapperGenerator.cs | 12 +++++-- .../XUnitWrapperLibrary/TestOutputRecorder.cs | 34 +++++++++++++++++++ .../Common/XUnitWrapperLibrary/TestSummary.cs | 16 ++++----- 4 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 0e9a1822c737d..b8aa63cba15fe 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -314,13 +314,15 @@ sealed class NoTestReporting : ITestReporterWrapper sealed class WrapperLibraryTestSummaryReporting : ITestReporterWrapper { - private string _summaryLocalIdentifier; + private readonly string _summaryLocalIdentifier; private readonly string _filterLocalIdentifier; + private readonly string _outputRecorderIdentifier; - public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string filterLocalIdentifier) + public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string filterLocalIdentifier, string outputRecorderIdentifier) { _summaryLocalIdentifier = summaryLocalIdentifier; _filterLocalIdentifier = filterLocalIdentifier; + _outputRecorderIdentifier = outputRecorderIdentifier; } public string WrapTestExecutionWithReporting(string testExecutionExpression, ITestInfo test) @@ -332,12 +334,13 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;"); builder.AppendLine("try {"); builder.AppendLine($"System.Console.WriteLine(\"Running test: {{0}}\", {test.TestNameExpression});"); + builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();"); builder.AppendLine(testExecutionExpression); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart);"); + builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());"); builder.AppendLine($"System.Console.WriteLine(\"Passed test: {{0}}\", {test.TestNameExpression});"); builder.AppendLine("}"); builder.AppendLine("catch (System.Exception ex) {"); - builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex);"); + builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());"); builder.AppendLine($"System.Console.WriteLine(\"Failed test: {{0}}\", {test.TestNameExpression});"); builder.AppendLine("}"); diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 22bd59ab82952..eb4295c06cc33 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -151,12 +151,15 @@ private static string GenerateFullTestRunner(ImmutableArray testInfos { // For simplicity, we'll use top-level statements for the generated Main method. StringBuilder builder = new(); - ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter"); builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};"))); builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = args.Length != 0 ? new XUnitWrapperLibrary.TestFilter(args[0]) : null;"); builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();"); builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();"); + builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); + builder.AppendLine("System.Console.SetOut(outputRecorder);"); + + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); foreach (ITestInfo test in testInfos) { @@ -175,7 +178,7 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI StringBuilder builder = new(); builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};"))); - + builder.AppendLine("try {"); builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests, ""{assemblyName}"", args.Length != 0 ? args[0] : null);"); builder.AppendLine("} catch(System.Exception ex) { System.Console.WriteLine(ex.ToString()); return 101; }"); @@ -183,8 +186,11 @@ private static string GenerateXHarnessTestRunner(ImmutableArray testI builder.AppendLine("static XUnitWrapperLibrary.TestSummary RunTests(XUnitWrapperLibrary.TestFilter filter)"); builder.AppendLine("{"); builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();"); - ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter"); builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = new();"); + builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);"); + builder.AppendLine("System.Console.SetOut(outputRecorder);"); + + ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder"); foreach (ITestInfo test in testInfos) { diff --git a/src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs b/src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs new file mode 100644 index 0000000000000..13b4c3d4ae52f --- /dev/null +++ b/src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace XUnitWrapperLibrary; + +public sealed class TestOutputRecorder : TextWriter +{ + private TextWriter _inner; + private ThreadLocal _testOutput = new(() => new StringBuilder()); + + public TestOutputRecorder(TextWriter inner) + { + _inner = inner; + } + + public override void Write(char value) + { + _inner.Write(value); + _testOutput.Value!.Append(value); + } + + public override Encoding Encoding => _inner.Encoding; + + public void ResetTestOutput() => _testOutput.Value!.Clear(); + + public string GetTestOutput() => _testOutput.Value!.ToString(); +} diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs index 5c1180fe30be0..a2f23bad9bbf8 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs @@ -9,7 +9,7 @@ namespace XUnitWrapperLibrary; public class TestSummary { - readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason); + readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output); public int PassedTests { get; private set; } = 0; public int FailedTests { get; private set; } = 0; @@ -19,22 +19,22 @@ public class TestSummary private DateTime _testRunStart = DateTime.Now; - public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration) + public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output) { PassedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null)); + _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null, output)); } - public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex) + public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output) { FailedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null)); + _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null, output)); } public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason) { SkippedTests++; - _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason)); + _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason, null)); } public string GetTestResultOutput(string assemblyName) @@ -72,7 +72,7 @@ public string GetTestResultOutput(string assemblyName) resultsFile.Append($@""); + resultsFile.AppendLine($@"result=""Fail"">"); } else if (test.SkipReason is not null) { @@ -80,7 +80,7 @@ public string GetTestResultOutput(string assemblyName) } else { - resultsFile.AppendLine(@" result=""Pass"" />"); + resultsFile.AppendLine($@" result=""Pass"">"); } }