Factor out CompilerAssert, TestFramework, ILChecker and Utilities (#9350

) * [WIP] Factor out CompilerAsserts, TestFramework, ILChecker to separate library * [WIP] +NUnit reference; Put Utilities.fs under module * Fixed tests to use new helpers library * Added <NoWarn>3186</NoWarn> to the FSharpSuite.Tests. * Added missing import for the TestHelpers. * Added missing solution entries for the project * Fix tests.fs to run tests with proper path * Fixed more paths in the tests * More fixes to test paths * More fixes to test paths (again)
dotnet · Jun 1, 2020 · aaeb058 · aaeb058
1 parent 0ae1389
commit aaeb058
Show file tree

Hide file tree

Showing 60 changed files with 1,085 additions and 961 deletions.
diff --git a/FSharp.sln b/FSharp.sln
@@ -24,6 +24,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiAnyCpu", "src\fsharp\fsi
 		{649FA588-F02E-457C-9FCF-87E46407481E} = {649FA588-F02E-457C-9FCF-87E46407481E}
 	EndProjectSection
 EndProject
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.TestsHelpers", "tests\FSharp.TestHelpers\FSharp.TestHelpers.fsproj", "{60D275B0-B14A-41CB-A1B2-E815A7448FCB} "
+EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharpSuite.Tests", "tests\fsharp\FSharpSuite.Tests.fsproj", "{C163E892-5BF7-4B59-AA99-B0E8079C67C4}"
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Compiler.UnitTests", "tests\FSharp.Compiler.UnitTests\FSharp.Compiler.UnitTests.fsproj", "{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}"
@@ -156,6 +158,18 @@ Global
 		{8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Release|x86.ActiveCfg = Release|Any CPU
 		{8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Release|x86.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|x86.Build.0 = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|Any CPU.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|Any CPU.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|x86.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|x86.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|x86.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|x86.Build.0 = Release|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|x86.ActiveCfg = Debug|Any CPU
@@ -276,6 +290,7 @@ Global
 		{649FA588-F02E-457C-9FCF-87E46407481E} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{D0E98C0D-490B-4C61-9329-0862F6E87645} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{8B3E283D-B5FE-4055-9D80-7E3A32F3967B} = {B8DDA694-7939-42E3-95E5-265C2217C142}
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{88E2D422-6852-46E3-A740-83E391DC7973} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}

diff --git a/VisualFSharp.sln b/VisualFSharp.sln
@@ -86,6 +86,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsiAnyCpu", "src\fsharp\fsi
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "fsi", "src\fsharp\fsi\fsi.fsproj", "{D0E98C0D-490B-4C61-9329-0862F6E87645}"
 EndProject
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.TestsHelpers", "tests\FSharp.TestHelpers\FSharp.TestHelpers.fsproj", "{60D275B0-B14A-41CB-A1B2-E815A7448FCB} "
+EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharpSuite.Tests", "tests\fsharp\FSharpSuite.Tests.fsproj", "{C163E892-5BF7-4B59-AA99-B0E8079C67C4}"
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Compiler.UnitTests", "tests\FSharp.Compiler.UnitTests\FSharp.Compiler.UnitTests.fsproj", "{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}"
@@ -528,6 +530,18 @@ Global
 		{D0E98C0D-490B-4C61-9329-0862F6E87645}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D0E98C0D-490B-4C61-9329-0862F6E87645}.Release|x86.ActiveCfg = Release|Any CPU
 		{D0E98C0D-490B-4C61-9329-0862F6E87645}.Release|x86.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Debug|x86.Build.0 = Debug|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|Any CPU.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|Any CPU.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|x86.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Proto|x86.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|x86.ActiveCfg = Release|Any CPU
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB}.Release|x86.Build.0 = Release|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4}.Debug|x86.ActiveCfg = Debug|Any CPU
@@ -964,6 +978,7 @@ Global
 		{649FA588-F02E-457C-9FCF-87E46407481E} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{8B3E283D-B5FE-4055-9D80-7E3A32F3967B} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{D0E98C0D-490B-4C61-9329-0862F6E87645} = {B8DDA694-7939-42E3-95E5-265C2217C142}
+		{60D275B0-B14A-41CB-A1B2-E815A7448FCB} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{C163E892-5BF7-4B59-AA99-B0E8079C67C4} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{88E2D422-6852-46E3-A740-83E391DC7973} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}

diff --git a/tests/fsharp/Compiler/CompilerAssert.fs → tests/FSharp.TestHelpers/CompilerAssert.fs b/tests/fsharp/Compiler/CompilerAssert.fs → tests/FSharp.TestHelpers/CompilerAssert.fs
@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
-[<AutoOpen>]
-module FSharp.Compiler.UnitTests.CompilerAssert
+namespace FSharp.TestHelpers
 
 open System
 open System.Diagnostics
@@ -20,7 +19,7 @@ open NUnit.Framework
 open System.Reflection.Emit
 open Microsoft.CodeAnalysis
 open Microsoft.CodeAnalysis.CSharp
-open FSharp.Compiler.UnitTests.Utilities
+open FSharp.TestHelpers.Utilities
 
 [<Sealed>]
 type ILVerifier (dllFilePath: string) =
@@ -56,8 +55,8 @@ type CompileOutput =
     | Library
     | Exe
 
-type CompilationReference = 
-    private 
+type CompilationReference =
+    private
     | CompilationReference of Compilation * staticLink: bool
     | TestCompilationReference of TestCompilation
 
@@ -254,11 +253,11 @@ let main argv = 0"""
             |> Array.map (fun info ->
                 (info.Severity, info.ErrorNumber, (info.StartLineAlternate - libAdjust, info.StartColumn + 1, info.EndLineAlternate - libAdjust, info.EndColumn + 1), info.Message))
 
-        let checkEqual k a b = 
-            if a <> b then 
+        let checkEqual k a b =
+            if a <> b then
                 Assert.AreEqual(a, b, sprintf "Mismatch in %s, expected '%A', got '%A'.\nAll errors:\n%A" k a b errors)
 
-        checkEqual "Errors"  (Array.length expectedErrors) errors.Length 
+        checkEqual "Errors"  (Array.length expectedErrors) errors.Length
 
         Array.zip errors expectedErrors
         |> Array.iter (fun (actualError, expectedError) ->
@@ -278,16 +277,16 @@ let main argv = 0"""
         let compilationRefs, deps =
             match cmpl with
             | Compilation(_, _, _, _, cmpls, _) ->
-                let compiledRefs =               
+                let compiledRefs =
                     cmpls
                     |> List.map (fun cmpl ->
                             match cmpl with
                             | CompilationReference (cmpl, staticLink) ->
                                 compileCompilationAux outputPath disposals ignoreWarnings cmpl, staticLink
-                            | TestCompilationReference (cmpl) -> 
+                            | TestCompilationReference (cmpl) ->
                                 let tmp = Path.Combine(outputPath, Path.ChangeExtension(Path.GetRandomFileName(), ".dll"))
-                                disposals.Add({ new IDisposable with 
-                                                    member _.Dispose() = 
+                                disposals.Add({ new IDisposable with
+                                                    member _.Dispose() =
                                                         try File.Delete tmp with | _ -> () })
                                 cmpl.EmitAsFile tmp
                                 (([||], tmp), []), false)
@@ -330,14 +329,14 @@ let main argv = 0"""
             match cmpl with
             | Compilation(source, _, _, _, _, _) -> source
 
-        let options = 
+        let options =
             match cmpl with
             | Compilation(_, _, _, options, _, _) -> options
 
         let nameOpt =
             match cmpl with
             | Compilation(_, _, _, _, _, nameOpt) -> nameOpt
-                    
+
         let disposal, res = compileDisposable outputPath isScript isExe (Array.append options compilationRefs) nameOpt source
         disposals.Add disposal
 
@@ -362,7 +361,7 @@ let main argv = 0"""
 
     static member CompileWithErrors(cmpl: Compilation, expectedErrors, ?ignoreWarnings) =
         let ignoreWarnings = defaultArg ignoreWarnings false
-        lock gate (fun () -> 
+        lock gate (fun () ->
             compileCompilation ignoreWarnings cmpl (fun ((errors, _), _) ->
                 assertErrors 0 ignoreWarnings errors expectedErrors))
 
@@ -374,7 +373,7 @@ let main argv = 0"""
         let beforeExecute = defaultArg beforeExecute (fun _ _ -> ())
         let newProcess = defaultArg newProcess false
         let onOutput = defaultArg onOutput (fun _ -> ())
-        lock gate (fun () -> 
+        lock gate (fun () ->
             compileCompilation ignoreWarnings cmpl (fun ((errors, outputFilePath), deps) ->
                 assertErrors 0 ignoreWarnings errors [||]
                 beforeExecute outputFilePath deps
@@ -447,9 +446,9 @@ let main argv = 0"""
 
             Assert.IsEmpty(typeCheckResults.Errors, sprintf "Type Check errors: %A" typeCheckResults.Errors)
 
-    static member TypeCheckWithErrorsAndOptionsAgainstBaseLine options (sourceFile: string) =
+    static member TypeCheckWithErrorsAndOptionsAgainstBaseLine options (sourceDirectory:string) (sourceFile: string) =
         lock gate <| fun () ->
-            let absoluteSourceFile = System.IO.Path.Combine(__SOURCE_DIRECTORY__, "..", sourceFile)
+            let absoluteSourceFile = System.IO.Path.Combine(sourceDirectory, sourceFile)
             let parseResults, fileAnswer =
                 checker.ParseAndCheckFileInProject(
                     sourceFile,
@@ -473,7 +472,7 @@ let main argv = 0"""
             let errorsActual =
                 typeCheckResults.Errors
                 |> Array.map (sprintf "%A")
-                |> String.concat "\n" 
+                |> String.concat "\n"
             File.WriteAllText(Path.ChangeExtension(absoluteSourceFile,"err"), errorsActual)
 
             Assert.AreEqual(errorsExpectedBaseLine.Replace("\r\n","\n"), errorsActual.Replace("\r\n","\n"))
@@ -489,7 +488,7 @@ let main argv = 0"""
                         { defaultProjectOptions with OtherOptions = Array.append options defaultProjectOptions.OtherOptions})
                     |> Async.RunSynchronously
 
-                if parseResults.Errors.Length > 0 then 
+                if parseResults.Errors.Length > 0 then
                     parseResults.Errors
                 else
 

diff --git a/tests/FSharp.TestHelpers/Directory.Build.props b/tests/FSharp.TestHelpers/Directory.Build.props
@@ -0,0 +1,9 @@
+<Project>
+
+  <PropertyGroup>
+    <UseFSharpProductVersion>true</UseFSharpProductVersion>
+  </PropertyGroup>
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
+
+</Project>
diff --git a/tests/FSharp.TestHelpers/FSharp.TestHelpers.fsproj b/tests/FSharp.TestHelpers/FSharp.TestHelpers.fsproj
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>net472;netcoreapp3.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix'">netcoreapp3.0</TargetFrameworks>
+    <RuntimeIdentifiers>win-x86;win-x64</RuntimeIdentifiers>
+    <AssetTargetFallback>$(AssetTargetFallback);portable-net45+win8+wp8+wpa81</AssetTargetFallback>
+    <ReferenceVsAssemblies>true</ReferenceVsAssemblies>
+    <OutputType>Library</OutputType>
+    <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
+    <Optimize>false</Optimize>
+    <Tailcalls>false</Tailcalls>
+    <OtherFlags>$(OtherFlags) --warnon:1182</OtherFlags>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\src\scripts\scriptlib.fsx">
+      <Link>scriptlib.fsx</Link>
+    </Compile>
+    <Compile Include="TestFramework.fs" />
+    <Compile Include="ILChecker.fs" />
+    <Compile Include="Utilities.fs" />
+    <Compile Include="CompilerAssert.fs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Compiler.Private\FSharp.Compiler.Private.fsproj" />
+    <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Core\FSharp.Core.fsproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Net.Compilers" Version="$(MicrosoftNetCompilersVersion)" />
+    <PackageReference Include="Microsoft.NETCore.ILDAsm" Version="$(MicrosoftNETCoreILDAsmVersion)" />
+    <PackageReference Include="Microsoft.NETCore.ILAsm" Version="$(MicrosoftNETCoreILAsmVersion)" />
+    <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
+    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
+    <PackageReference Include="System.Runtime.Loader" Version="$(SystemRuntimeLoaderVersion)" />
+    <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.Common" Version="$(MicrosoftCodeAnalysisWorkspacesCommonVersion)" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisCSharpVersion)" />
+    <PackageReference Include="Microsoft.CodeAnalysis.Test.Resources.Proprietary" Version="$(MicrosoftCodeAnalysisTestResourcesProprietaryVersion)" />
+    <PackageReference Include="NUnit" Version="$(NUnitVersion)" />
+  </ItemGroup>
+</Project>
diff --git a/tests/fsharp/Compiler/ILChecker.fs → tests/FSharp.TestHelpers/ILChecker.fs b/tests/fsharp/Compiler/ILChecker.fs → tests/FSharp.TestHelpers/ILChecker.fs
@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
-namespace FSharp.Compiler.UnitTests
+namespace FSharp.TestHelpers
 
 open System
 open System.IO

diff --git a/tests/fsharp/test-framework.fs → tests/FSharp.TestHelpers/TestFramework.fs b/tests/fsharp/test-framework.fs → tests/FSharp.TestHelpers/TestFramework.fs
@@ -1,3 +1,5 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
 module TestFramework
 
 open Microsoft.Win32
@@ -7,7 +9,6 @@ open System.Text.RegularExpressions
 open Scripting
 open NUnit.Framework
 
-
 [<RequireQualifiedAccess>]
 module Commands =