Capture and record output from tests from Console.Out into the test r…

…esults (#63606) * Capture and record output from tests from Console.Out into the test results. * Simplify construction. * Fix nullability * Fix constructor visibility
dotnet · Jan 12, 2022 · ef7ff07 · ef7ff07
1 parent ad1b5b8
commit ef7ff07
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 15 deletions.
diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs
@@ -314,13 +314,15 @@ sealed class NoTestReporting : ITestReporterWrapper
 
 sealed class WrapperLibraryTestSummaryReporting : ITestReporterWrapper
 {
-    private string _summaryLocalIdentifier;
+    private readonly string _summaryLocalIdentifier;
     private readonly string _filterLocalIdentifier;
+    private readonly string _outputRecorderIdentifier;
 
-    public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string filterLocalIdentifier)
+    public WrapperLibraryTestSummaryReporting(string summaryLocalIdentifier, string filterLocalIdentifier, string outputRecorderIdentifier)
     {
         _summaryLocalIdentifier = summaryLocalIdentifier;
         _filterLocalIdentifier = filterLocalIdentifier;
+        _outputRecorderIdentifier = outputRecorderIdentifier;
     }
 
     public string WrapTestExecutionWithReporting(string testExecutionExpression, ITestInfo test)
@@ -332,12 +334,13 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe
         builder.AppendLine($"System.TimeSpan testStart = stopwatch.Elapsed;");
         builder.AppendLine("try {");
         builder.AppendLine($"System.Console.WriteLine(\"Running test: {{0}}\", {test.TestNameExpression});");
+        builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();");
         builder.AppendLine(testExecutionExpression);
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart);");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());");
         builder.AppendLine($"System.Console.WriteLine(\"Passed test: {{0}}\", {test.TestNameExpression});");
         builder.AppendLine("}");
         builder.AppendLine("catch (System.Exception ex) {");
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex);");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());");
         builder.AppendLine($"System.Console.WriteLine(\"Failed test: {{0}}\", {test.TestNameExpression});");
         builder.AppendLine("}");
 

diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs
@@ -151,12 +151,15 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
     {
         // For simplicity, we'll use top-level statements for the generated Main method.
         StringBuilder builder = new();
-        ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter");
         builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};")));
 
         builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = args.Length != 0 ? new XUnitWrapperLibrary.TestFilter(args[0]) : null;");
         builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
         builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();");
+        builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);");
+        builder.AppendLine("System.Console.SetOut(outputRecorder);");
+
+        ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder");
 
         foreach (ITestInfo test in testInfos)
         {
@@ -175,16 +178,19 @@ private static string GenerateXHarnessTestRunner(ImmutableArray<ITestInfo> testI
         StringBuilder builder = new();
         builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};")));
 
-        
+
         builder.AppendLine("try {");
         builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests, ""{assemblyName}"", args.Length != 0 ? args[0] : null);");
         builder.AppendLine("} catch(System.Exception ex) { System.Console.WriteLine(ex.ToString()); return 101; }");
 
         builder.AppendLine("static XUnitWrapperLibrary.TestSummary RunTests(XUnitWrapperLibrary.TestFilter filter)");
         builder.AppendLine("{");
         builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
-        ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter");
         builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = new();");
+        builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);");
+        builder.AppendLine("System.Console.SetOut(outputRecorder);");
+
+        ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder");
 
         foreach (ITestInfo test in testInfos)
         {

diff --git a/src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs b/src/tests/Common/XUnitWrapperLibrary/TestOutputRecorder.cs
@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading;
+
+namespace XUnitWrapperLibrary;
+
+public sealed class TestOutputRecorder : TextWriter
+{
+    private TextWriter _inner;
+    private ThreadLocal<StringBuilder> _testOutput = new(() => new StringBuilder());
+
+    public TestOutputRecorder(TextWriter inner)
+    {
+        _inner = inner;
+    }
+
+    public override void Write(char value)
+    {
+        _inner.Write(value);
+        _testOutput.Value!.Append(value);
+    }
+
+    public override Encoding Encoding => _inner.Encoding;
+
+    public void ResetTestOutput() => _testOutput.Value!.Clear();
+
+    public string GetTestOutput() => _testOutput.Value!.ToString();
+}
diff --git a/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs b/src/tests/Common/XUnitWrapperLibrary/TestSummary.cs
@@ -9,7 +9,7 @@ namespace XUnitWrapperLibrary;
 
 public class TestSummary
 {
-    readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason);
+    readonly record struct TestResult(string Name, string ContainingTypeName, string MethodName, TimeSpan Duration, Exception? Exception, string? SkipReason, string? Output);
 
     public int PassedTests { get; private set; } = 0;
     public int FailedTests { get; private set; } = 0;
@@ -19,22 +19,22 @@ public class TestSummary
 
     private DateTime _testRunStart = DateTime.Now;
 
-    public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration)
+    public void ReportPassedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string output)
     {
         PassedTests++;
-        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null));
+        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, null, output));
     }
 
-    public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex)
+    public void ReportFailedTest(string name, string containingTypeName, string methodName, TimeSpan duration, Exception ex, string output)
     {
         FailedTests++;
-        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null));
+        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, ex, null, output));
     }
 
     public void ReportSkippedTest(string name, string containingTypeName, string methodName, TimeSpan duration, string reason)
     {
         SkippedTests++;
-        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason));
+        _testResults.Add(new TestResult(name, containingTypeName, methodName, duration, null, reason, null));
     }
 
     public string GetTestResultOutput(string assemblyName)
@@ -72,15 +72,15 @@ public string GetTestResultOutput(string assemblyName)
             resultsFile.Append($@"<test name=""{test.Name}"" type=""{test.ContainingTypeName}"" method=""{test.MethodName}"" time=""{test.Duration.TotalSeconds:F6}"" ");
             if (test.Exception is not null)
             {
-                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{test.Exception.Message}]]></message><stack-trace><![CDATA[{test.Exception.StackTrace}]]></stack-trace></failure></test>");
+                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{test.Exception.Message}]]></message><stack-trace><![CDATA[{test.Exception.StackTrace}]]></stack-trace></failure><output><![CDATA[{test.Output}]]></output></test>");
             }
             else if (test.SkipReason is not null)
             {
                 resultsFile.AppendLine($@"result=""Skip""><reason><![CDATA[{test.SkipReason}]]></reason></test>");
             }
             else
             {
-                resultsFile.AppendLine(@" result=""Pass"" />");
+                resultsFile.AppendLine($@" result=""Pass""><output><![CDATA[{test.Output}]]></output></test>");
             }
         }