[build] extract .NET workloads to a temp folder (#1027)

Our build currently emits some confusing warnings such as: Microsoft.NET.Sdk.ImportWorkloads.props(14,3): warning MSB4243: The NuGet-based SDK resolver failed to run because NuGet assemblies could not be located. Check your installation of MSBuild or set the environment variable "MSBUILD_NUGET_PATH" to the folder that contains the required NuGet assemblies. Could not find file 'D:\repos\dotnet\maui\bin\dotnet\sdk-manifests\6.0.100\microsoft.build.notargets\WorkloadManifest.json'. Only the very last line is actually relevant: Could not find file 'D:\repos\dotnet\maui\bin\dotnet\sdk-manifests\6.0.100\microsoft.build.notargets\WorkloadManifest.json'. This is because we are extracting the workload `.nupkg` files directly into `bin/dotnet/sdk-manifests/6.0.100` with the `$NUGET_PACKAGES` environment variable. Because we are using NuGet to do this, we hit some weird problems like extra directories getting created by NuGet: * `microsoft.build.notargets` * `microsoft.netcore.platforms` * `netstandard.library` A better fix for this problem is to extract the files into: .\bin\temp\ ...and then we move the `WorkloadManifest.*` files to the locations we need. Any extra files can just be left in `temp`, so they will not cause problems in future builds. To clean things up more, I removed the `.manifest-6.0.100` string from the directory names, so they match what is in dotnet/installer. I also moved a few MSBuild properties to `Directory.Build.props` so they could be used throughout the entire repo down the road. This will assist in making .NET MAUI a workload.
dotnet · May 17, 2021 · e02ec40 · e02ec40
1 parent 57bf776
commit e02ec40
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 25 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -31,7 +31,12 @@
   </PropertyGroup>
 
   <PropertyGroup>
-
+    <DotNetOutputPath>$(MSBuildThisFileDirectory)bin/</DotNetOutputPath>
+    <DotNetTempDirectory>$(DotNetOutputPath)temp/</DotNetTempDirectory>
+    <DotNetDirectory>$(DotNetOutputPath)dotnet/</DotNetDirectory>
+    <DotNetToolPath>$(DotNetDirectory)dotnet</DotNetToolPath>
+    <DotNetPacksDirectory>$(DotNetDirectory)packs/</DotNetPacksDirectory>
+    <DotNetSdkManifestsDirectory>$(DotNetDirectory)sdk-manifests/$(DotNetPreviewVersionBand)/</DotNetSdkManifestsDirectory>
     <_MauiBuildTasksLocation>$(_MauiBuildTasksLocation)</_MauiBuildTasksLocation>
     <_MauiBuildTasksLocation Condition="'$(_MauiBuildTasksLocation)' == ''">$(MSBuildThisFileDirectory).nuspec\</_MauiBuildTasksLocation>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

diff --git a/src/DotNet/DotNet.csproj b/src/DotNet/DotNet.csproj
@@ -2,23 +2,18 @@
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <IsPackable>false</IsPackable>
-    <MauiOutputPath>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)../../bin/'))</MauiOutputPath>
-    <DotNetDirectory>$(MauiOutputPath)dotnet/</DotNetDirectory>
-    <DotNetToolPath>$(DotNetDirectory)dotnet</DotNetToolPath>
-    <DotNetPacksDirectory>$(DotNetDirectory)packs/</DotNetPacksDirectory>
-    <DotNetSdkManifestsDirectory>$(DotNetDirectory)sdk-manifests/$(DotNetPreviewVersionBand)/</DotNetSdkManifestsDirectory>
   </PropertyGroup>
   <PropertyGroup Condition="$([MSBuild]::IsOSPlatform('windows'))">
     <DotNetInstallScriptUrl>https://dot.net/v1/dotnet-install.ps1</DotNetInstallScriptUrl>
     <DotNetInstallScriptName>dotnet-install.ps1</DotNetInstallScriptName>
-    <DotNetInstallScriptPath>$(MauiOutputPath)$(DotNetInstallScriptName)</DotNetInstallScriptPath>
+    <DotNetInstallScriptPath>$(DotNetOutputPath)$(DotNetInstallScriptName)</DotNetInstallScriptPath>
     <DotNetInstallCommand>&amp; '$(DotNetInstallScriptPath)' -Version $(MicrosoftDotnetSdkInternalPackageVersion) -InstallDir '$(DotNetDirectory)' -Verbose</DotNetInstallCommand>
     <DotNetInstallCommand>powershell -Command &quot;$(DotNetInstallCommand)&quot;</DotNetInstallCommand>
   </PropertyGroup>
   <PropertyGroup Condition="$([MSBuild]::IsOSPlatform('osx'))">
     <DotNetInstallScriptUrl>https://dot.net/v1/dotnet-install.sh</DotNetInstallScriptUrl>
     <DotNetInstallScriptName>dotnet-install.sh</DotNetInstallScriptName>
-    <DotNetInstallScriptPath>$(MauiOutputPath)$(DotNetInstallScriptName)</DotNetInstallScriptPath>
+    <DotNetInstallScriptPath>$(DotNetOutputPath)$(DotNetInstallScriptName)</DotNetInstallScriptPath>
     <DotNetInstallCommand>sh '$(DotNetInstallScriptPath)' --version $(MicrosoftDotnetSdkInternalPackageVersion) --install-dir '$(DotNetDirectory)' --verbose</DotNetInstallCommand>
   </PropertyGroup>
 
@@ -45,7 +40,7 @@
       Outputs="$(DotNetInstallScriptPath)">
     <DownloadFile
         SourceUrl="$(DotNetInstallScriptUrl)"
-        DestinationFolder="$(MauiOutputPath)"
+        DestinationFolder="$(DotNetOutputPath)"
         DestinationFileName="$(DotNetInstallScriptName)"
     />
   </Target>
@@ -59,10 +54,6 @@
   </Target>
 
   <ItemGroup>
-    <!-- These are extra packs that get extracted that should be deleted -->
-    <_PacksToRemove Include="microsoft.build.notargets" />
-    <_PacksToRemove Include="microsoft.netcore.platforms" />
-    <_PacksToRemove Include="netstandard.library" />
     <!-- These are past workload names -->
     <_PacksToRemove Include="Microsoft.NET.Workload.Android" />
     <_PacksToRemove Include="Microsoft.NET.Workload.MacCatalyst" />
@@ -74,45 +65,48 @@
   <Target Name="_AcquireWorkloads"
       Inputs="$(_Inputs);Dependencies/Workloads.csproj"
       Outputs="$(DotNetSdkManifestsDirectory).stamp">
+    <RemoveDir Directories="@(_PacksToRemove->'$(DotNetSdkManifestsDirectory)%(Identity)')" />
     <Exec
         Command="&quot;$(DotNetToolPath)&quot; restore &quot;$(MSBuildThisFileDirectory)Dependencies/Workloads.csproj&quot; -bl:$(PackageOutputPath)/DotNetWorkloads.binlog"
-        EnvironmentVariables="NUGET_PACKAGES=$(DotNetSdkManifestsDirectory);DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1"
+        EnvironmentVariables="NUGET_PACKAGES=$(DotNetTempDirectory);DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1"
     />
     <!--
-      NOTE: Workloads need to go in dotnet/sdk-manifests/6.0.100/Microsoft.NET.Sdk.*/
+      NOTE: Workloads need to go in dotnet/sdk-manifests/6.0.100/Microsoft.NET.*/
       Workloads only contain two files: data/WorkloadManifest.targets & data/WorkloadManifest.json
-      We have to move everything up two directories, for things to land in the right location.
+      We downloaded these to $(DotNetTempDirectory), so we could move files individually.
     -->
     <ItemGroup>
-      <_WorkloadFiles Include="$(DotNetSdkManifestsDirectory)Microsoft.NET.*/*/data/WorkloadManifest.*" />
-      <_WorkloadDirectoriesToDelete Include="@(_WorkloadFiles->'%(RootDir)%(Directory)'->Distinct())" />
+      <_WorkloadFiles Include="$(DotNetTempDirectory)Microsoft.NET.*/*/data/WorkloadManifest.*" />
     </ItemGroup>
-    <Copy SourceFiles="@(_WorkloadFiles)" DestinationFiles="@(_WorkloadFiles->'%(RootDir)%(Directory)../../%(FileName)%(Extension)')" />
-    <RemoveDir Directories="@(_WorkloadDirectoriesToDelete);@(_PacksToRemove->'$(DotNetSdkManifestsDirectory)%(Identity)')" />
+    <CopyWorkloadFiles
+        Files="@(_WorkloadFiles)"
+        WorkloadDirectory="$(DotNetSdkManifestsDirectory)"
+        VersionBand="$(DotNetPreviewVersionBand)"
+    />
 
     <!-- In order to support older MSBuild versions, change the version string to an integer -->
     <ReplaceText
-        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.android.manifest-$(DotNetPreviewVersionBand)/WorkloadManifest.json"
+        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.android/WorkloadManifest.json"
         OldValue="&quot;version&quot;: &quot;$(MicrosoftAndroidSdkWindowsPackageVersion)&quot;"
         NewValue="&quot;version&quot;: 6"
     />
     <ReplaceText
-        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.maccatalyst.manifest-$(DotNetPreviewVersionBand)/WorkloadManifest.json"
+        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.maccatalyst/WorkloadManifest.json"
         OldValue="&quot;version&quot;: &quot;$(MicrosoftMacCatalystSdkPackageVersion)&quot;"
         NewValue="&quot;version&quot;: 6"
     />
     <ReplaceText
-        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.ios.manifest-$(DotNetPreviewVersionBand)/WorkloadManifest.json"
+        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.ios/WorkloadManifest.json"
         OldValue="&quot;version&quot;: &quot;$(MicrosoftiOSSdkPackageVersion)&quot;"
         NewValue="&quot;version&quot;: 6"
     />
     <ReplaceText
-        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.tvos.manifest-$(DotNetPreviewVersionBand)/WorkloadManifest.json"
+        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.tvos/WorkloadManifest.json"
         OldValue="&quot;version&quot;: &quot;$(MicrosofttvOSSdkPackageVersion)&quot;"
         NewValue="&quot;version&quot;: 6"
     />
     <ReplaceText
-        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.macos.manifest-$(DotNetPreviewVersionBand)/WorkloadManifest.json"
+        Input="$(DotNetSdkManifestsDirectory)microsoft.net.sdk.macos/WorkloadManifest.json"
         OldValue="&quot;version&quot;: &quot;$(MicrosoftmacOSSdkPackageVersion)&quot;"
         NewValue="&quot;version&quot;: 6"
     />
@@ -131,6 +125,32 @@
     <Touch Files="$(DotNetPacksDirectory).stamp" AlwaysCreate="true" />
   </Target>
 
+  <UsingTask TaskName="CopyWorkloadFiles"
+      TaskFactory="RoslynCodeTaskFactory"
+      AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">
+    <ParameterGroup>
+      <Files ParameterType="System.String[]" Required="true" />
+      <WorkloadDirectory ParameterType="System.String" Required="true" />
+      <VersionBand ParameterType="System.String" Required="true" />
+    </ParameterGroup>
+    <Task>
+      <Using Namespace="System.IO" />
+      <Code Type="Fragment" Language="cs">
+        <![CDATA[
+          // This is only in C#, because the equivalent MSBuild XML would not be pretty.
+          foreach (var file in Files)
+          {
+              var name = Directory.GetParent (file).Parent.Parent.Name;
+              var directory = Path.Combine(WorkloadDirectory, name.Replace($".manifest-{VersionBand}", ""));
+              Directory.CreateDirectory(directory);
+              var destination = Path.Combine(directory, Path.GetFileName(file));
+              Log.LogMessage($"Copying {file} to {destination}");
+              File.Copy(file, destination, overwrite: true);
+          }
+        ]]>
+      </Code>
+    </Task>
+  </UsingTask>
   <UsingTask TaskName="ReplaceText"
       TaskFactory="RoslynCodeTaskFactory"
       AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">