Actually merge from main

.editorconfig

@@ -26,3 +26,8 @@ dotnet_diagnostic.MSML_ExtendBaseTestClass.severity = none
 # The MSML_RelaxTestNaming suppressor for VSTHRD200 is not active for CodeAnalyzer.Tests, so we disable it altogether.
 # VSTHRD200: Use "Async" suffix for async methods
 dotnet_diagnostic.VSTHRD200.severity = none
+
+# Xml project files
+[*.{csproj}]
+indent_size = 2
+charset = utf-8

.gitignore

@@ -4,7 +4,8 @@
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
 
 # Tool Runtime Dir
-/[Tt]ools/
+/.dotnet/
+/.packages/
 
 # User-specific files
 *.suo
@@ -226,7 +227,7 @@ ClientBin/
 *.publishsettings
 orleans.codegen.cs
 
-# Including strong name files can present a security risk 
+# Including strong name files can present a security risk
 # (https://github.com/github/gitignore/pull/2483#issue-259490424)
 #*.snk
 
@@ -323,7 +324,7 @@ __pycache__/
 # OpenCover UI analysis results
 OpenCover/
 
-# Azure Stream Analytics local run output 
+# Azure Stream Analytics local run output
 ASALocalRun/
 
 # MSBuild Binary and Structured Log

.vsts-dotnet-ci.yml

@@ -6,7 +6,7 @@ resources:
   containers:
   - container: CentosContainer
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
-  
+
   - container: UbuntuContainer
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
 
@@ -29,7 +29,7 @@ jobs:
         _targetFramework: netcoreapp3.1
     innerLoop: true
     pool:
-      name:  Hosted Ubuntu 1604
+      name: Hosted Ubuntu 1604
 
 - template: /build/ci/job-template.yml
   parameters:
@@ -38,7 +38,7 @@ jobs:
     container: UbuntuContainer
     innerLoop: true
     pool:
-      name:  Hosted Ubuntu 1604
+      name: Hosted Ubuntu 1604
 
 - template: /build/ci/job-template.yml
   parameters:

Directory.Build.props

@@ -1,117 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
 <Project>
-  <!-- Directory.Build.props contains the common build settings for all projects in the repo. -->
 
-  <Import Project="build/BranchInfo.props" />
-  <Import Project="build/Dependencies.props" />
+  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+
+  <Import Project="eng/BranchInfo.props" />
+  <Import Project="eng/ExternalBenchmarkDataFiles.props" />
+  <Import Project="eng/TensorflowMetaFiles.props" />
 
   <PropertyGroup>
 	  <CreateSymbolicLinksForPublishFilesIfPossible>true</CreateSymbolicLinksForPublishFilesIfPossible>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <Copyright>$(CopyrightNetFoundation)</Copyright>
+    <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+    <DebugType>portable</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
   <PropertyGroup>
     <Configuration Condition="'$(Configuration)'==''">Debug</Configuration>
     <Configurations>Debug;Release;Debug-netcoreapp3_1;Release-netcoreapp3_1;Debug-netfx;Release-netfx</Configurations>
     <Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
     <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">x64</TargetArchitecture>
     <NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
     <PlatformConfig>$(Platform).$(Configuration)</PlatformConfig>
+    <StrongNameKeyId>Open</StrongNameKeyId>
   </PropertyGroup>
 
   <PropertyGroup>
-    <RestoreSources>
-      https://api.nuget.org/v3/index.json;
-      https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
-      https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
-      https://dotnet.myget.org/F/roslyn-analyzers/api/v3/index.json;
-      https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json;
-      https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-testdata/nuget/v3/index.json;
-    </RestoreSources>
-  </PropertyGroup>
-
-  <!-- Common repo directories -->
-  <PropertyGroup>
-    <RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
-    <SourceDir>$(RepoRoot)src/</SourceDir>
+    <PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
     <PkgDir>$(RepoRoot)pkg/</PkgDir>
-
-    <!-- Output directories -->
-    <BinDir Condition="'$(BinDir)'==''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'bin'))</BinDir>
-    <BaseOutputPath Condition="'$(BaseOutputPath)'==''">$(BinDir)</BaseOutputPath>
-    <ObjDir Condition="'$(ObjDir)'==''">$([MSBuild]::NormalizeDirectory('$(BinDir)', 'obj'))</ObjDir>
-    <RootIntermediateOutputPath Condition="'$(RootIntermediateOutputPath)'==''">$(ObjDir)</RootIntermediateOutputPath>
-
-    <IntermediateOutputRootPath Condition="'$(IntermediateOutputRootPath)' == ''">$(RootIntermediateOutputPath)$(PlatformConfig)\</IntermediateOutputRootPath>
-    <IntermediateOutputPath Condition="'$(IntermediateOutputPath)' == ''">$(IntermediateOutputRootPath)$(MSBuildProjectName)\</IntermediateOutputPath>
-    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)' == ''">$(IntermediateOutputPath)</BaseIntermediateOutputPath>
-
-    <OutputPath Condition="'$(OutputPath)'==''">$(BaseOutputPath)$(PlatformConfig)\$(MSBuildProjectName)\</OutputPath>
-
-    <PackageAssetsPath>$(ObjDir)/packages/</PackageAssetsPath>
-
-    <PackageOutputPath Condition="'$(PackageOutputPath)'=='' and '$(NonShippingPackage)' == 'true'">$(BinDir)packages_noship/</PackageOutputPath>
-    <PackageOutputPath Condition="'$(PackageOutputPath)'==''">$(BinDir)packages/</PackageOutputPath>
-
-    <NativeOutputPath>$(BaseOutputPath)$(NativeTargetArchitecture).$(Configuration)\Native\</NativeOutputPath>
-
-    <!-- Input Directories -->
-    <PackagesDir>$(DotNetRestorePackagesPath)</PackagesDir>
-    <PackagesDir Condition="'$(PackagesDir)'==''">$(RepoRoot)packages/</PackagesDir>
-    <RestorePackagesPath>$(PackagesDir)</RestorePackagesPath>
-    <ToolsDir Condition="'$(ToolsDir)'==''">$(RepoRoot)Tools/</ToolsDir>
-  </PropertyGroup>
-
-  <Import Project="$(ToolsDir)BuildVersion.targets" 
-          Condition="Exists('$(ToolsDir)BuildVersion.targets')" />
-
-  <!-- Version properties -->
-  <PropertyGroup>
-    <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
-	<BuildNumberMajor Condition="'$(BuildNumberMajor)' == ''">00001</BuildNumberMajor>
-    <BuildNumberMinor Condition="'$(BuildNumberMinor)' == ''">0</BuildNumberMinor>
-    <AssemblyFileVersion Condition="'$(AssemblyFileVersion)'==''">$(MajorVersion).$(MinorVersion).$(BuildNumberMajor).$(BuildNumberMinor)</AssemblyFileVersion>
-
-    <StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
-    <IncludeBuildNumberInPackageVersion Condition="'$(IncludeBuildNumberInPackageVersion)' == '' and '$(StabilizePackageVersion)' != 'true'">true</IncludeBuildNumberInPackageVersion>
-
-    <VersionSuffix Condition="'$(IncludeBuildNumberInPackageVersion)' == 'true'">$(BuildNumberMajor)-$(BuildNumberMinor)</VersionSuffix>
-    <NoPackageAnalysis>true</NoPackageAnalysis>
-  </PropertyGroup>
-
-  <!-- SourceLink properties used by dotnet/buildtools - need to be set before importing $(ToolsDir)versioning.props -->
-  <PropertyGroup>
-    <UseSourceLink>true</UseSourceLink>
-    <GitHubRepositoryName>machinelearning</GitHubRepositoryName>
-  </PropertyGroup>
-
-  <!--
-  Source code control properties used by the .NET Core SDK to inject SCC info into the NuGet package.
-  In future versions, these will be used for SourceLink and to generate AssemblyInfo.
-  -->
-  <PropertyGroup>
-    <PrivateRepositoryUrl>https://github.com/dotnet/$(GitHubRepositoryName)</PrivateRepositoryUrl>
-    <PublishRepositoryUrl>true</PublishRepositoryUrl>
-    <SourceRevisionId>$(LatestCommit)</SourceRevisionId>
-  </PropertyGroup>
-
-  <Import Project="$(ToolsDir)versioning.props" 
-          Condition="Exists('$(ToolsDir)versioning.props') and '$(DisableImportVersioningProps)' != 'true'" />
-
-  <!-- Language configuration -->
-  <PropertyGroup>
-    <LangVersion Condition="'$(MSBuildProjectExtension)'=='.csproj'">8.0</LangVersion>
-    <LangVersion Condition="'$(MSBuildProjectExtension)'=='.fsproj'">4.7</LangVersion>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-  </PropertyGroup>
-
-  <PropertyGroup>
-    <RunningOnUnix Condition="'$(OS)'!='Windows_NT'">true</RunningOnUnix>
-  </PropertyGroup>
-
-  <!-- Signing properties -->
-  <PropertyGroup>
-    <AssemblyOriginatorKeyFile Condition="'$(AssemblyOriginatorKeyFile)' == ''">$(ToolsDir)Open.snk</AssemblyOriginatorKeyFile>
-    <SignAssembly>true</SignAssembly>
-    <PublicSign Condition="'$(OS)' != 'Windows_NT'">true</PublicSign>
   </PropertyGroup>
 
   <!-- Need to explicitly set these properties for the -netcoreapp3_1 or -netfx configurations becuase they are typically based off 'Debug' or 'Release' configs -->
@@ -125,4 +45,34 @@
     <Optimize>true</Optimize>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!-- Optional: Publish the repository URL in the built .nupkg (in the NuSpec <Repository> element) -->
+    <PublishRepositoryUrl>true</PublishRepositoryUrl>
+
+    <!-- Optional: Embed source files that are not tracked by the source control manager in the PDB -->
+    <EmbedUntrackedSources>true</EmbedUntrackedSources>
+
+    <!-- Optional: Build symbol package (.snupkg) to distribute the PDB containing Source Link -->
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
+    <!--
+      When building using source-build the process is:
+      - Newtonsoft.Json versions 9.0.1 and 12.0.2 are built by source-build
+      - Version 12.0.2 is written to Version.props
+      - Arcade needs to use 9.0.1 so we need to override Version.props value here
+    -->
+    <NewtonsoftJsonVersion>9.0.1</NewtonsoftJsonVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(DotNetBuildOffline)' == 'true'">
+    <!--
+      Arcade has a special version prop for CodeAnalysis.CSharp in GenFacades
+      to try to match the version loaded by msbuild.  In the offline build, this
+      is simply the source-built version.
+    -->
+    <MsbuildTaskMicrosoftCodeAnalysisCSharpVersion>$(MicrosoftCodeAnalysisCSharpVersion)</MsbuildTaskMicrosoftCodeAnalysisCSharpVersion>
+  </PropertyGroup>
 </Project>

Directory.Build.targets

@@ -1,51 +1,83 @@
-<Project InitialTargets="CheckForBuildTools">
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
-  <ItemGroup>
-    <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="16.5.132" PrivateAssets="all" />
+  <ItemGroup Condition="'$(IsUnitTestProject)' == 'true'">
+    <PackageReference Include="coverlet.msbuild" Version="2.9.0" PrivateAssets="all" />
   </ItemGroup>
 
-  <Target Name="CheckForBuildTools">
-    <Error Condition="!Exists('$(ToolsDir)')"
-           Text="The tools directory [$(ToolsDir)] does not exist. Please run build in the root of the repo to ensure the tools are installed before attempting to build an individual project." />
-  </Target>
+  <Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory">
+    <PropertyGroup>
+		<LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
+		<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
+		<LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
+		<LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
 
-  <Target Name="CopyNativeAssemblies"
-          BeforeTargets="PrepareForRun">
+		<TargetArchitecture Condition="'$(Platform)' == ''">x64</TargetArchitecture>
+		<NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
+		<BinDir Condition="'$(BinDir)'==''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin'))</BinDir>
+		<NativeOutputPath>$(BinDir)Native\$(NativeTargetArchitecture).$(Configuration)\</NativeOutputPath>
 
-    <PropertyGroup>
-      <LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
-      <LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
-      <LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
-      <LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <NativeAssemblyReference>
-        <FullAssemblyPath>$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReference.Identity)$(LibExtension)</FullAssemblyPath>
-      </NativeAssemblyReference>
-    </ItemGroup>
-
-    <Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
-          DestinationFolder="$(OutputPath)"
-          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
-          Retries="$(CopyRetryCount)"
-          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
-          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
-          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
-      <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
-    </Copy>
+		<Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
+		<PlatformConfig>$(Platform).$(Configuration)</PlatformConfig>
+		<OutputPath Condition="'$(OutputPath)'==''">$(BinDir)$(MSBuildProjectName)\Debug</OutputPath>
+	</PropertyGroup>
 
+		<ItemGroup>
+		<NativeAssemblyReference>
+			<FullAssemblyPath>$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReference.Identity)$(LibExtension)</FullAssemblyPath>
+		</NativeAssemblyReference>
+		</ItemGroup>
+
+		<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
+			DestinationFolder="$(OutDir)"
+			OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+			Retries="$(CopyRetryCount)"
+			RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+			UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+			UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
+		<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
+		</Copy>
   </Target>
-
-  <Import Project="$(ToolsDir)/versioning.targets" Condition="Exists('$(ToolsDir)/versioning.targets')" />
+  <PropertyGroup Condition="'$(Coverage)' == 'true'">
+    <!-- https://github.com/tonerdo/coverlet/issues/363 -->
+    <DeterministicSourcePaths>false</DeterministicSourcePaths>
+
+    <!-- https://github.com/tonerdo/coverlet/issues/618 -->
+    <IncludeTestAssembly>true</IncludeTestAssembly>
 
- <!-- Workaround: AssemblyInfo.cs is not embedded -->
- <!-- https://github.com/dotnet/sourcelink/issues/572 -->
-  <PropertyGroup>
-    <TargetFrameworkMonikerAssemblyAttributesPath>$([System.IO.Path]::Combine('$(IntermediateOutputPath)','$(TargetFrameworkMoniker).AssemblyAttributes$(DefaultLanguageSourceExtension)'))</TargetFrameworkMonikerAssemblyAttributesPath>
+    <CollectCoverage>true</CollectCoverage>
+    <SingleHit>true</SingleHit>
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <CoverletOutputFormat>opencover</CoverletOutputFormat>
+    <CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\coverage.opencover.xml</CoverletOutput>
+    <Include></Include>
+    <Exclude></Exclude>
+    <ExcludeByAttribute>ExcludeFromCodeCoverage</ExcludeByAttribute>
+    <ExcludeByFile></ExcludeByFile>
   </PropertyGroup>
-  <ItemGroup>
-    <EmbeddedFiles Include="$(GeneratedAssemblyInfoFile)"/>
-  </ItemGroup>
 
-</Project>
+  <Target Name="OuterInstrumentModulesNoBuild" BeforeTargets="RunTests" Condition="'$(IsUnitTestProject)' == 'true' AND '$(TargetFramework)' == ''">
+    <MSBuild
+      Projects="$(MSBuildProjectFullPath)"
+      Targets="InnerInstrumentModulesNoBuild"
+      Properties="TargetFramework=%(_TargetFramework.Identity)" />
+  </Target>
+
+  <Target Name="OuterGenerateCoverageResult" BeforeTargets="Test" Condition="'$(IsUnitTestProject)' == 'true' AND '$(TargetFramework)' == ''">
+    <MSBuild
+      Projects="$(MSBuildProjectFullPath)"
+      Targets="InnerGenerateCoverageResult"
+      Properties="TargetFramework=%(_TargetFramework.Identity)" />
+  </Target>
+
+  <Target Name="InnerInstrumentModulesNoBuild"
+          BeforeTargets="RunTests"
+          DependsOnTargets="InstrumentModules"
+          Condition="'$(IsUnitTestProject)' == 'true' AND '$(TargetFramework)' != '' AND '$(CollectCoverage)' == 'true'" />
+
+  <Target Name="InnerGenerateCoverageResult"
+          BeforeTargets="Test"
+          DependsOnTargets="GenerateCoverageResult"
+          Condition="'$(IsUnitTestProject)' == 'true' AND '$(TargetFramework)' != '' AND '$(CollectCoverage)' == 'true'" />
+</Project>