Improve incremental build times of the libraries subset

Directory.Build.props

@@ -304,7 +304,6 @@
 
   <PropertyGroup>
     <CustomBeforeNoTargets>$(RepositoryEngineeringDir)NoTargetsSdk.BeforeTargets.targets</CustomBeforeNoTargets>
-    <CustomAfterTraversalProps>$(RepositoryEngineeringDir)TraversalSdk.AfterProps.props</CustomAfterTraversalProps>
     <CustomAfterTraversalTargets>$(RepositoryEngineeringDir)TraversalSdk.AfterTargets.targets</CustomAfterTraversalTargets>
   </PropertyGroup>
 </Project>

docs/coding-guidelines/project-guidelines.md

@@ -6,10 +6,13 @@ In order to work in the dotnet/runtime repo you must first run build.cmd/sh from
 - Restore tools
 - Restore external dependencies
  - CoreCLR - Copy to `bin\runtime\$(BuildTargetFramework)-$(TargetOS)-$(Configuration)-$(TargetArchitecture)`
-- Build targeting pack
- - Build src\libraries\ref.proj which builds all references assembly projects. For reference assembly project information see [ref](#ref)
-- Build product
- - Build src\libraries\src.proj which builds all the source library projects. For source library project information see [src](#src).
+- Build shared framework projects
+ - Build src\libraries\sfx.proj which builds all shared framework projects.
+- Build out of band projects
+ - Build src\libraries\oob.proj which builds all the out-of-band (OOB) projects.
+
+For reference assembly project information see [ref](#ref)
+For source library project information see [src](#src)
 
 # Build Pivots
 Below is a list of all the various options we pivot the project builds on:

eng/NoTargetsSdk.BeforeTargets.targets

@@ -1,9 +1,8 @@
 <Project>
 
   <PropertyGroup>
-    <!-- NoTargets SDK needs a TFM set. Use the latest .NETCoreApp TFM that is supported by the SDK.
-         Only set a default if the project isn't multi-targeting. -->
-    <TargetFramework Condition="'$(TargetFramework)' == '' and '$(TargetFrameworks)' == ''">$(NetCoreAppToolCurrent)</TargetFramework>
+    <!-- NoTargets SDK needs a TFM set. Set a default if the project doesn't multi target. -->
+    <TargetFramework Condition="'$(TargetFramework)' == '' and '$(TargetFrameworks)' == ''">$(NetCoreAppCurrent)</TargetFramework>
   </PropertyGroup>
 
 </Project>

eng/Subsets.props

@@ -71,7 +71,7 @@
     <DefaultLibrariesSubsets Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
                                         '$(BuildTargetFramework)' == '' or
                                         '$(BuildAllConfigurations)' == 'true'">libs.native+</DefaultLibrariesSubsets>
-    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.sfx+libs.oob</DefaultLibrariesSubsets>
     <DefaultLibrariesSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultLibrariesSubsets)+libs.pretest</DefaultLibrariesSubsets>
 
     <DefaultHostSubsets>host.native+host.tools</DefaultHostSubsets>
@@ -144,8 +144,10 @@
     <!-- Libs -->
     <SubsetName Include="Libs" Description="The libraries native part, refs and source assemblies, test infra and packages, but NOT the tests (use Libs.Tests to request those explicitly). Equivalent to: $(DefaultLibrariesSubsets)" />
     <SubsetName Include="Libs.Native" Description="The native libraries used in the shared framework." />
-    <SubsetName Include="Libs.Ref" Description="The managed reference libraries." />
-    <SubsetName Include="Libs.Src" Description="The managed implementation libraries." />
+    <SubsetName Include="Libs.Sfx" Description="The managed shared framework libraries." />
+    <SubsetName Include="Libs.Oob" Description="The managed out-of-band libraries." />
+    <SubsetName Include="Libs.Ref" OnDemand="true" Description="The managed reference libraries." />
+    <SubsetName Include="Libs.Src" OnDemand="true" Description="The managed implementation libraries." />
     <SubsetName Include="Libs.PreTest" Description="Test assets which are necessary to run tests." />
     <SubsetName Include="Libs.Packages" Description="The projects that produce NuGet packages from libraries." />
     <SubsetName Include="Libs.Tests" OnDemand="true" Description="The test projects. Note that building this doesn't execute tests: you must also pass the '-test' argument." />
@@ -322,12 +324,20 @@
     <ProjectToBuild Include="$(SharedNativeRoot)libs\build-native.proj" Category="libs" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+libs.ref+'))">
-    <ProjectToBuild Include="$(LibrariesProjectRoot)ref.proj" Category="libs" />
+  <ItemGroup Condition="$(_subset.Contains('+libs.ref+')) or $(_subset.Contains('+libs.src+')) or $(_subset.Contains('+libs.sfx+'))">
+    <ProjectToBuild Include="$(LibrariesProjectRoot)sfx.proj"
+                    Category="libs"
+                    Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
+                               '$(BuildTargetFramework)' == '' or
+                               '$(BuildAllConfigurations)' == 'true'">
+      <AdditionalProperties Condition="$(_subset.Contains('+libs.ref+'))">%(AdditionalProperties);RefOnly=true</AdditionalProperties>
+    </ProjectToBuild>
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+libs.src+'))">
-    <ProjectToBuild Include="$(LibrariesProjectRoot)src.proj" Category="libs" />
+  <ItemGroup Condition="$(_subset.Contains('+libs.ref+')) or $(_subset.Contains('+libs.src+')) or $(_subset.Contains('+libs.oob+'))">
+    <ProjectToBuild Include="$(LibrariesProjectRoot)oob.proj" Category="libs">
+      <AdditionalProperties Condition="$(_subset.Contains('+libs.ref+'))">%(AdditionalProperties);RefOnly=true</AdditionalProperties>
+    </ProjectToBuild>
   </ItemGroup>
 
   <ItemGroup Condition="$(_subset.Contains('+mono.wasmruntime+'))">

eng/TraversalSdk.AfterTargets.targets

@@ -9,10 +9,17 @@
     <VSTestDependsOn>$(VSTestDependsOn);ResolveP2PReferences</VSTestDependsOn>
     <PackDependsOn>$(PackDependsOn);ResolveP2PReferences</PackDependsOn>
     <PublishDependsOn>$(PublishDependsOn);ResolveP2PReferences</PublishDependsOn>
+    <GetTargetPathDependsOn>ResolveP2PReferences</GetTargetPathDependsOn>
+    <!-- Filter out ProjectReferences which aren't compatible with the project's TargetFramework. -->
+    <OmitIncompatibleProjectReferences>true</OmitIncompatibleProjectReferences>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(FilterTraversalProjectReferences)' == 'true'">
-    <ProjectReference Update="@(ProjectReference)" SkipGetTargetFrameworkProperties="false" />
+    <!-- Override the Traversal SDK setting as filtering relies on the TargetFrameworkProperties being fetched
+         and don't flow the BuildTargetFramework property down. -->
+    <ProjectReference Update="@(ProjectReference)"
+                      SkipGetTargetFrameworkProperties="false"
+                      UndefineProperties="%(UndefineProperties);BuildTargetFramework"  />
   </ItemGroup>
 
-</Project>
+</Project>

eng/generators.targets

@@ -13,8 +13,7 @@
          - references System.Runtime.InteropServices -->
     <EnabledGenerators Include="DllImportGenerator"
                        Condition="'$(EnableDllImportGenerator)' == ''
-                        and '$(IsFrameworkSupportFacade)' != 'true'
-                        and '$(IsSourceProject)' == 'true'
+                        and '$(IsRuntimeAssembly)' == 'true'
                         and '$(MSBuildProjectExtension)' == '.csproj'
                         and (
                           ('@(Reference)' != ''
@@ -25,8 +24,7 @@
                             and '$(DisableImplicitAssemblyReferences)' == 'false'))" />
     <EnabledGenerators Include="DllImportGenerator"
                        Condition="'$(EnableDllImportGenerator)' == ''
-                        and '$(IsFrameworkSupportFacade)' != 'true'
-                        and '$(IsSourceProject)' == 'true'
+                        and '$(IsRuntimeAssembly)' == 'true'
                         and '$(MSBuildProjectExtension)' == '.csproj'
                         and ('$(TargetFrameworkIdentifier)' == '.NETStandard' or '$(TargetFrameworkIdentifier)' == '.NETFramework' or ('$(TargetFrameworkIdentifier)' == '.NETCoreApp' and $([MSBuild]::VersionLessThan($(TargetFrameworkVersion), '$(NetCoreAppCurrentVersion)'))))" />
   </ItemGroup>

eng/illink.targets

@@ -198,14 +198,22 @@
     </ItemGroup>
   </Target>
 
-  <!-- ILLink.Tasks arguments common to runs for both individual libraries and for the entire runtime pack -->
-  <Target Name="SetCommonILLinkArgs">
+  <Target Name="PrepareForAssembliesTrim">
+    <!-- ILLink.Tasks arguments common to runs for both individual libraries and for the entire runtime pack -->
     <PropertyGroup>
       <!-- don't remove attributes after build, our tooling is not ready for that -->
       <ILLinkArgs>$(ILLinkArgs) --ignore-link-attributes true</ILLinkArgs>
       <!-- ignore unresolved references -->
       <ILLinkArgs>$(ILLinkArgs) --skip-unresolved true</ILLinkArgs>
     </PropertyGroup>
+
+    <!-- When running from Desktop MSBuild, DOTNET_HOST_PATH is not set.
+         In this case, explicitly specify the path to the dotnet host. -->
+    <PropertyGroup Condition="'$(DOTNET_HOST_PATH)' == ''">
+      <!-- This is defined when building in Visual Studio. -->
+      <_DotNetHostDirectory>$(NetCoreRoot)</_DotNetHostDirectory>
+      <_DotNetHostFileName>$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetHostFileName>
+    </PropertyGroup>
   </Target>
 
   <!-- ILLinkTrimAssembly
@@ -215,7 +223,7 @@
   <UsingTask TaskName="ILLink" AssemblyFile="$(ILLinkTasksAssembly)" Condition="'$(ILLinkTasksAssembly)' != ''" />
   <Target Name="ILLinkTrimAssembly"
           Condition="'$(ILLinkTrimAssembly)' == 'true'"
-          DependsOnTargets="SetCommonILLinkArgs">
+          DependsOnTargets="PrepareForAssembliesTrim">
     <PropertyGroup>
       <!-- default action for assemblies with IsTrimmable attribute -->
       <ILLinkArgs>$(ILLinkArgs) --trim-mode skip</ILLinkArgs>
@@ -285,21 +293,12 @@
       <ILLinkArgs Condition="@(_DependencyDirectories->Count()) > 0">$(ILLinkArgs) -d @(_DependencyDirectories->'"%(Identity)"', ' -d ')</ILLinkArgs>
     </PropertyGroup>
 
-    <!-- When running from Desktop MSBuild, DOTNET_HOST_PATH is not set.
-         In this case, explicitly specify the path to the dotnet host. -->
-    <PropertyGroup Condition=" '$(DOTNET_HOST_PATH)' == '' ">
-      <!-- This is defined when building in Visual Studio. -->
-      <_DotNetHostDirectory>$(NetCoreRoot)</_DotNetHostDirectory>
-      <_DotNetHostFileName>$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetHostFileName>
-    </PropertyGroup>
-
     <ILLink AssemblyPaths=""
             RootAssemblyNames="@(ILLinkTrimInputAssembly)"
             OutputDirectory="$(ILLinkTrimOutputPath)"
             ExtraArgs="$(ILLinkArgs)"
             ToolExe="$(_DotNetHostFileName)"
             ToolPath="$(_DotNetHostDirectory)" />
-
   </Target>
 
   <!-- ILLink reporting.

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -74,7 +74,7 @@ jobs:
       - ${{ each variable in parameters.variables }}:
         - ${{ variable }}
       - name: liveRuntimeBuildParams
-        value: 'libs.ref -c Release -ci'
+        value: 'libs.sfx+libs.oob /p:RefOnly=true -c Release -ci'
       - name: compilerArg
         value: ''
       - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}:

eng/pipelines/runtime-official.yml

@@ -57,7 +57,7 @@ stages:
   - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
     - template: /eng/common/templates/job/source-index-stage1.yml
       parameters:
-        sourceIndexBuildCommand: build.cmd -subset libs.ref+libs.src -binarylog -os Linux -ci
+        sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os Linux -ci
 
   #
   # Build CoreCLR

eng/pipelines/runtime-richnav.yml

@@ -37,7 +37,7 @@ jobs:
       richCodeNavigationEnvironment: "production"
       richCodeNavigationLanguage: "csharp"
       timeoutInMinutes: 240
-      buildArgs: -s libs.ref+libs.src -c debug -allConfigurations
+      buildArgs: -s libs.sfx+libs.oob -allConfigurations
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:

eng/referenceAssemblies.props

@@ -1,14 +1,9 @@
 <Project>
-  <PropertyGroup Condition="'$(BuildAllProjects)' == 'true' and
-                            !$(BuildTargetFramework.StartsWith('netstandard')) and
-                            !$(BuildTargetFramework.StartsWith('net4'))">
-    <AdditionalBuildTargetFrameworks>$(AdditionalBuildTargetFrameworks);netstandard2.0;netstandard2.1</AdditionalBuildTargetFrameworks>
 
+  <PropertyGroup>
     <!-- Reference assemblies are special and don't initialize fields or have empty finalizers, etc. -->
     <RunAnalyzers>false</RunAnalyzers>
-  </PropertyGroup>
 
-  <PropertyGroup>
     <!-- disable warnings about unused fields -->
     <NoWarn>$(NoWarn);CS0169;CS0649;CS8618</NoWarn>
 
@@ -30,4 +25,5 @@
       <_Parameter1_IsLiteral>true</_Parameter1_IsLiteral>
     </AssemblyAttribute>
   </ItemGroup>
+
 </Project>

eng/references.targets

@@ -16,19 +16,17 @@
 
   <ItemGroup Condition="'@(ProjectReference)' != ''">
     <_coreLibProjectReference Include="@(ProjectReference->WithMetadataValue('Identity', '$(CoreLibProject)'))" />
-    <ProjectReference Update="@(_coreLibProjectReference)">
-      <!-- Don't flow TargetFramework and Platform to use same inputs and outputs as the CoreLib's build as part of the runtime. -->
-      <UndefineProperties>$(UndefineProperties);TargetFramework;Platform</UndefineProperties>
+    <ProjectReference Update="@(_coreLibProjectReference)"
+                      Private="false">
       <SetConfiguration Condition="'$(RuntimeFlavor)' == 'CoreCLR' and
                                    '$(Configuration)' != '$(CoreCLRConfiguration)'">Configuration=$(CoreCLRConfiguration)</SetConfiguration>
       <SetConfiguration Condition="'$(RuntimeFlavor)' == 'Mono' and
                                    '$(Configuration)' != '$(MonoConfiguration)'">Configuration=$(MonoConfiguration)</SetConfiguration>
-      <Private>false</Private>
     </ProjectReference>
     <!-- If a CoreLib ProjectReference is present, make all P2P assets non transitive. -->
-    <ProjectReference Update="@(ProjectReference)"
+    <ProjectReference Update="@(ProjectReference->WithMetadataValue('PrivateAssets', ''))"
                       PrivateAssets="all"
-                      Condition="'@(_coreLibProjectReference)' != ''" />
+                      Condition="'$(IsSourceProject)' == 'true' and '@(_coreLibProjectReference)' != ''" />
   </ItemGroup>
 
   <!-- Disable TargetArchitectureMismatch warning when we reference CoreLib as it is platform specific. -->
@@ -78,8 +76,19 @@
 
   <Target Name="ValidateReferenceAssemblyProjectReferences"
           AfterTargets="ResolveReferences"
-          Condition="'$(IsReferenceAssembly)' == 'true'">
+          Condition="'$(IsReferenceAssembly)' == 'true' and
+                     '$(SkipValidateReferenceAssemblyProjectReferences)' != 'true'">
     <Error Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference' and '%(ReferencePath.IsReferenceAssembly)' != 'true' and '%(ReferencePath.ReferenceAssembly)' == ''"
            Text="Reference assemblies must only reference other reference assemblies and '%(ReferencePath.ProjectReferenceOriginalItemSpec)' is not a reference assembly project and does not set 'ProduceReferenceAssembly'." />
   </Target>
+
+  <!-- An opt-in target to trim out private assemblies from the ref assembly ReferencePath. -->
+  <Target Name="TrimOutPrivateAssembliesFromReferencePath"
+          Condition="'$(CompileUsingReferenceAssemblies)' == 'true' and '$(TrimOutPrivateAssembliesFromReferencePath)' == 'true'"
+          AfterTargets="FindReferenceAssembliesForReferences">
+    <ItemGroup>
+      <ReferencePathWithRefAssemblies Remove="@(ReferencePathWithRefAssemblies)"
+                                      Condition="$(NetCoreAppLibraryNoReference.Contains('%(Filename);'))" />
+    </ItemGroup>
+  </Target>
 </Project>

eng/resolveContract.targets

@@ -28,7 +28,10 @@
           DependsOnTargets="ResolveProjectReferences"
           AfterTargets="GetTargetPathWithTargetPlatformMoniker">
     <ItemGroup>
-      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContract)" />
+      <!-- Allow to point to a different reference project than what GenFacades uses to generate the type forwards. -->
+      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContractOverride)" />
+      <TargetPathWithTargetPlatformMoniker ReferenceAssembly="@(ResolvedMatchingContract)"
+                                           Condition="'@(ResolvedMatchingContractOverride)' == ''" />
     </ItemGroup>
   </Target>
 
@@ -38,7 +41,7 @@
           Condition="'@(ProjectReference)' != '' and '@(_ResolvedProjectReferencePaths)' != ''">
     <!-- If we have a ProjectReference to CoreLib, we need to compile against implementation assemblies. -->
     <PropertyGroup Condition="@(_ResolvedProjectReferencePaths->AnyHaveMetadataValue('MSBuildSourceProjectFile', '$(CoreLibProject)'))">
-      <CompileUsingReferenceAssemblies>false</CompileUsingReferenceAssemblies>
+      <CompileUsingReferenceAssemblies Condition="'$(CompileUsingReferenceAssemblies)' == ''">false</CompileUsingReferenceAssemblies>
     </PropertyGroup>
     <!-- Clear the ReferenceAssembly attribute on resolved P2Ps that set SkipUseReferenceAssembly to true. -->
     <ItemGroup>

eng/targetingpacks.targets

@@ -146,6 +146,13 @@
       <_targetingPackReferenceExclusion Include="@(DefaultReferenceExclusion)" />
     </ItemGroup>
 
+    <!-- Filter out shims from the targeting pack references as an opt-in. -->
+    <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' and
+                          '$(SkipTargetingPackShimReferences)' == 'true'">
+      <_targetingPackReferenceExclusion Include="@(NetFxReference)" />
+      <_targetingPackReferenceExclusion Include="netstandard" />
+    </ItemGroup>
+
     <ItemGroup>
       <_targetingPackReferenceWithProjectName Include="@(Reference->WithMetadataValue('ExternallyResolved', 'true')->Metadata('Filename'))"
                                               OriginalIdentity="%(Identity)" />

eng/versioning.targets

@@ -107,7 +107,10 @@
   <Target Name="AddOSPlatformAttributes"
           BeforeTargets="GenerateAssemblyInfo"
           AfterTargets="PrepareForBuild"
-          Condition="'$(TargetPlatformIdentifier)' == '' and '$(IsTestProject)' != 'true' and '$(TargetFrameworkIdentifier)' != '.NETFramework'">
+          Condition="'$(TargetPlatformIdentifier)' == '' and
+                     '$(IsTestProject)' != 'true' and
+                     '$(TargetFrameworkIdentifier)' != '.NETFramework'
+                     and '$(AddOSPlatformAttributes)' != 'false'">
     <!-- Defensively de-dupe the values -->
     <ItemGroup>
       <_unsupportedOSPlatforms Include="$(UnsupportedOSPlatforms)" />
@@ -133,12 +136,15 @@
     </ItemGroup>
   </Target>
 
-  <!-- Removes assembly level attributes from test projects. -->
-  <Target Name="RemoveSupportedOSPlatformAttributeFromTestProjects"
+  <!-- Remove assembly level attributes from certain projects.
+       Use a target for that until https://github.com/dotnet/sdk/issues/14836 is implemented. -->
+  <Target Name="RemoveSupportedOSTargetPlatformAttributeFromProjects"
           AfterTargets="GetAssemblyAttributes"
-          Condition="'$(IsTestProject)' == 'true'">
+          BeforeTargets="CreateGeneratedAssemblyInfoInputsCacheFile">
     <ItemGroup>
-      <AssemblyAttribute Remove="System.Runtime.Versioning.SupportedOSPlatformAttribute" />
+      <AssemblyAttribute Remove="System.Runtime.Versioning.SupportedOSPlatformAttribute"
+                         Condition="'$(IsTestProject)' == 'true' or '$(AddOSPlatformAttributes)' == 'false'" />
+      <!-- Don't include target platform attributes, since we use the target platform to represent RIDs instead. -->
       <AssemblyAttribute Remove="System.Runtime.Versioning.TargetPlatformAttribute" />
     </ItemGroup>
   </Target>