src/tests tree test xunit-based source generated runner (#60846)

Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
Co-authored-by: Jan Kotas <jkotas@microsoft.com>
Co-authored-by: Santiago Fernandez Madero <safern@microsoft.com>
Co-authored-by: Tomas <trylek@microsoft.com>

src/tests/Common/Assert.cs

@@ -12,6 +12,12 @@
 
 namespace Xunit
 {
+    /// <summary>
+    /// A copy of the Xunit.FactAttribute type for assemblies that reference System.Private.CoreLib directly.
+    /// </summary>
+    public sealed class FactAttribute : Attribute
+    {}
+
     /// <summary>
     ///    A collection of helper classes to test various conditions within
     /// unit tests. If the condition being tested is not met, an exception

src/tests/Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj

@@ -15,6 +15,11 @@
     <Compile Include="Utilities.cs" />
     <Compile Include="HostPolicyMock.cs" />
     <Compile Include="XPlatformUtils.cs" />
+    <Compile Include="CoreClrConfigurationDetection.cs" />
+    <Compile Include="OutOfProcessTest.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="..\Coreclr.TestWrapper\CoreclrTestWrapperLib.cs" Link="CoreclrTestWrapperLib.cs" />
   </ItemGroup>
 
   <ItemGroup>

src/tests/Common/CoreCLRTestLibrary/CoreClrConfigurationDetection.cs

@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+using System;
+using System.Globalization;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Security;
+using System.Text;
+
+namespace TestLibrary;
+
+public static class CoreClrConfigurationDetection
+{
+    public static bool IsJitStress => !string.Equals(GetEnvironmentVariableValue("JitStress"), "0", StringComparison.InvariantCulture);
+    public static bool IsJitStressRegs => !string.Equals(GetEnvironmentVariableValue("JitStressRegs"), "0", StringComparison.InvariantCulture);
+    public static bool IsJitMinOpts => string.Equals(GetEnvironmentVariableValue("JITMinOpts"), "1", StringComparison.InvariantCulture);
+    public static bool IsTailCallStress => string.Equals(GetEnvironmentVariableValue("TailcallStress"), "1", StringComparison.InvariantCulture);
+    public static bool IsZapDisable => string.Equals(GetEnvironmentVariableValue("ZapDisable"), "1", StringComparison.InvariantCulture);
+    public static bool IsGCStress3 => CompareGCStressModeAsLower(GetEnvironmentVariableValue("GCStress"), "0x3", "3");
+    public static bool IsGCStressC => CompareGCStressModeAsLower(GetEnvironmentVariableValue("GCStress"), "0xC", "C");
+
+    public static bool IsGCStress => GetEnvironmentVariableValue("GCStress") != string.Empty;
+
+    public static bool IsCheckedRuntime => AssemblyConfigurationEquals("Checked");
+    public static bool IsReleaseRuntime => AssemblyConfigurationEquals("Release");
+    public static bool IsDebugRuntime => AssemblyConfigurationEquals("Debug");
+
+    public static bool IsStressTest =>
+        IsGCStress ||
+        IsZapDisable ||
+        IsTailCallStress ||
+        IsJitStressRegs ||
+        IsJitStress ||
+        IsJitMinOpts;
+
+    private static string GetEnvironmentVariableValue(string name) =>
+        Environment.GetEnvironmentVariable("DOTNET_" + name) ?? Environment.GetEnvironmentVariable("COMPlus_" + name) ?? "0";
+
+    private static bool AssemblyConfigurationEquals(string configuration)
+    {
+        AssemblyConfigurationAttribute assemblyConfigurationAttribute = typeof(string).Assembly.GetCustomAttribute<AssemblyConfigurationAttribute>();
+
+        return assemblyConfigurationAttribute != null &&
+            string.Equals(assemblyConfigurationAttribute.Configuration, configuration, StringComparison.InvariantCulture);
+    }
+
+    private static bool CompareGCStressModeAsLower(string value, string first, string second)
+    {
+        value = value.ToLowerInvariant();
+        return string.Equals(value, first.ToLowerInvariant(), StringComparison.InvariantCulture) ||
+            string.Equals(value, second.ToLowerInvariant(), StringComparison.InvariantCulture) ||
+            string.Equals(value, "0xf", StringComparison.InvariantCulture) ||
+            string.Equals(value, "f", StringComparison.InvariantCulture);
+    }
+}

src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs

@@ -0,0 +1,132 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using CoreclrTestLib;
+using Xunit;
+
+namespace TestLibrary
+{
+    public static class OutOfProcessTest
+    {
+        internal static bool runningInWindows;
+        internal static string reportBase;
+        internal static string testBinaryBase;
+        internal static string helixUploadRoot;
+
+        static OutOfProcessTest()
+        {
+            reportBase = Directory.GetCurrentDirectory();
+            testBinaryBase = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
+            helixUploadRoot = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
+            if (!String.IsNullOrEmpty(helixUploadRoot))
+            {
+                reportBase = Path.Combine(Path.GetFullPath(helixUploadRoot), "Reports");
+            }
+
+            if (String.IsNullOrEmpty(reportBase))
+            {
+                reportBase = Path.Combine(testBinaryBase, "Reports");
+            }
+            else
+            {
+                reportBase = Path.GetFullPath(reportBase);
+            }
+        }
+
+        public static void RunOutOfProcessTest(string basePath, string assemblyPath)
+        {
+            int ret = -100;
+            string outputFile = System.IO.Path.GetFullPath(reportBase + assemblyPath + "output.txt");
+            string errorFile = System.IO.Path.GetFullPath(reportBase + assemblyPath + "error.txt");
+            string outputDir = System.IO.Path.GetDirectoryName(outputFile)!;
+            string testExecutable = null;
+            Exception infraEx = null;
+
+            try
+            {
+                CoreclrTestWrapperLib wrapper = new CoreclrTestWrapperLib();
+
+                if (OperatingSystem.IsWindows())
+                {
+                    testExecutable = Path.Combine(basePath, Path.ChangeExtension(assemblyPath, ".cmd"));
+                }
+                else
+                {
+                    testExecutable = Path.Combine(basePath, Path.ChangeExtension(assemblyPath.Replace("\\", "/"), ".sh"));
+                }
+
+                System.IO.Directory.CreateDirectory(reportBase + Path.GetDirectoryName(assemblyPath));
+
+                ret = wrapper.RunTest(testExecutable, outputFile, errorFile, Assembly.GetEntryAssembly()!.FullName!, testBinaryBase, outputDir);
+            }
+            catch (Exception ex)
+            {
+                infraEx = ex;
+            }
+
+            if (infraEx != null)
+            {
+                Assert.True(false, "Test Infrastructure Failure: " + infraEx.ToString());
+            }
+            else
+            {
+                List<string> testOutput = new List<string>();
+
+                try
+                {
+                    testOutput.AddRange(System.IO.File.ReadAllLines(errorFile));
+                }
+                catch (Exception ex)
+                {
+                    testOutput.Add("Unable to read error file: " + errorFile);
+                    testOutput.Add(ex.ToString());
+                }
+
+                testOutput.Add(string.Empty);
+                testOutput.Add("Return code:      " + ret);
+                testOutput.Add("Raw output file:      " + outputFile);
+                testOutput.Add("Raw output:");
+
+                try
+                {
+                    testOutput.AddRange(System.IO.File.ReadAllLines(outputFile));
+                }
+                catch (Exception ex)
+                {
+                    testOutput.Add("Unable to read output file: " + outputFile);
+                    testOutput.Add(ex.ToString());
+                }
+
+                testOutput.Add("To run the test:");
+                testOutput.Add("Set up CORE_ROOT and run.");
+                testOutput.Add("> " + testExecutable);
+
+                var unicodeControlCharsRegex = new Regex("%5C%5Cp{C}+");
+
+                // Remove all characters that have no visual or spatial representation.
+                for (int i = 0; i < testOutput.Count; i++)
+                {
+                    string line = testOutput[i];
+                    line = unicodeControlCharsRegex.Replace(line, string.Empty);
+                    testOutput[i] = line;
+                }
+
+                foreach (string line in testOutput)
+                {
+                    Console.WriteLine(line);
+                }
+
+                Assert.True(ret == CoreclrTestWrapperLib.EXIT_SUCCESS_CODE, string.Join(Environment.NewLine, testOutput));
+            }
+        }
+    }
+}

src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 //
+#nullable disable
 
 using System;
 using System.Collections.Generic;
@@ -196,7 +197,7 @@ public class CoreclrTestWrapperLib
     {
         public const int EXIT_SUCCESS_CODE = 0;
         public const string TIMEOUT_ENVIRONMENT_VAR = "__TestTimeout";
-        
+
         // Default timeout set to 10 minutes
         public const int DEFAULT_TIMEOUT_MS = 1000 * 60 * 10;
 
@@ -289,7 +290,7 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
             Debug.Assert(outputFile != errorFile);
 
             int exitCode = -100;
-            
+
             // If a timeout was given to us by an environment variable, use it instead of the default
             // timeout.
             string environmentVar = Environment.GetEnvironmentVariable(TIMEOUT_ENVIRONMENT_VAR);
@@ -387,7 +388,5 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
 
             return exitCode;
         }
-
-
     }
 }

src/tests/Common/Directory.Build.props

@@ -1,4 +1,8 @@
 <Project>
+  <PropertyGroup>
+    <IsTestsCommonProject>true</IsTestsCommonProject>
+  </PropertyGroup>
+
   <!-- SDK Style projects auto-magically include this file. -->
   <Import Project="..\Directory.Build.props" />
 </Project>

src/tests/Common/ILTestRunner/ILTestRunner.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsMergedTestRunnerAssembly>true</IsMergedTestRunnerAssembly>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(RepoRoot)/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
+    <ProjectReference Include="$(RepoRoot)/src/tests/Common/XUnitWrapperLibrary/XUnitWrapperLibrary.csproj" />
+  </ItemGroup>
+
+  <Import Project="$(RepoRoot)/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.props" />
+
+  <ItemGroup>
+    <ProjectReference Include="$(RepoRoot)\src\tests\JIT\IL_Conformance\Old\directed\AutoInit.ilproj" />
+  </ItemGroup>
+</Project>