Support cross-architecture compilation in ILC (#591)

* Support cross-architecture compilation in ILC
* Fix paths for CMake and build.sh
* Fix slashes in #include's path
* Replace 'git restore' with 'git checkout'
* Use 'exit /b 1' consistently in build.cmd

NuGet.config

@@ -14,12 +14,11 @@
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+    <add key="dotnet-experimental" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json" />
     <add key="dotnet5" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
     <add key="dotnet5-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json" />
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
     <add key="dotnet6-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json" />
-    <!-- Restore source for the ObjectWriter package - https://github.com/dotnet/runtimelab/issues/63 -->
-    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />

eng/Subsets.props

@@ -48,7 +48,7 @@
   <PropertyGroup>
     <DefaultCoreClrSubsets>clr.runtime+clr.jit+clr.alljits+linuxdac+clr.corelib+clr.tools+clr.nativecorelib+clr.packages</DefaultCoreClrSubsets>
 
-    <DefaultNativeAotSubsets>clr.jit+nativeaot.tools+nativeaot.libs</DefaultNativeAotSubsets>
+    <DefaultNativeAotSubsets>clr.jit+clr.alljits+nativeaot.tools+nativeaot.libs</DefaultNativeAotSubsets>
 
     <DefaultMonoSubsets Condition="'$(MonoEnableLLVM)' == 'true' and '$(MonoLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
     <DefaultMonoSubsets Condition="'$(MonoAOTEnableLLVM)' == 'true' and '$(MonoAOTLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
@@ -235,17 +235,17 @@
     <ProjectToBuild Include="$(CoreClrProjectRoot)nativeaot\Test.CoreLib\src\Test.CoreLib.csproj" Category="clr" />
   </ItemGroup>
 
+  <ItemGroup Condition="$(_subset.Contains('+nativeaot.objwriter+'))">
+    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ObjWriter\objwriter.proj" Category="clr" />
+  </ItemGroup>
+
   <ItemGroup Condition="$(_subset.Contains('+nativeaot.tools+'))">
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.Build.Tasks\ILCompiler.Build.Tasks.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\repro\repro.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)nativeaot\BuildIntegration\BuildIntegration.proj" Category="clr" />
   </ItemGroup>
 
-  <ItemGroup Condition="$(_subset.Contains('+nativeaot.objwriter+'))">
-    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ObjWriter\objwriter.proj" Category="clr" />
-  </ItemGroup>
-
   <!-- Mono sets -->
   <ItemGroup Condition="$(_subset.Contains('+mono.llvm+')) or $(_subset.Contains('+mono.aotcross+')) or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'Android' or '$(TargetOS)' == 'Browser'">
     <ProjectToBuild Include="$(MonoProjectRoot)llvm\llvm-init.proj" Category="mono" />

eng/pipelines/runtimelab.yml

@@ -103,7 +103,7 @@ stages:
         isOfficialBuild: ${{ variables.isOfficialBuild }}
         timeoutInMinutes: 90
         testGroup: innerloop
-        buildArgs: -s nativeaot+libs+nativeaot.packages -c release
+        buildArgs: -s nativeaot.objwriter+nativeaot+libs+nativeaot.packages -c release
         extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
         extraStepsParameters:
           uploadIntermediateArtifacts: ${{ variables.isOfficialBuild }}

src/coreclr/tools/Common/JitInterface/JitConfigProvider.cs

@@ -44,7 +44,6 @@ public static void Initialize(
             NativeLibrary.SetDllImportResolver(typeof(CorInfoImpl).Assembly, (libName, assembly, searchPath) =>
             {
                 IntPtr libHandle = IntPtr.Zero;
-#if READYTORUN
                 if (libName == CorInfoImpl.JitLibrary)
                 {
                     if (!string.IsNullOrEmpty(jitPath))
@@ -56,9 +55,6 @@ public static void Initialize(
                         libHandle = NativeLibrary.Load("clrjit_" + GetTargetSpec(target), assembly, searchPath);
                     }
                 }
-#else
-                Debug.Assert(jitPath == null);
-#endif
                 if (libName == CorInfoImpl.JitSupportLibrary)
                 {
                     libHandle = NativeLibrary.Load("jitinterface_" + RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(), assembly, searchPath);

src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj

@@ -9,7 +9,7 @@
     <PlatformTarget>AnyCPU</PlatformTarget>
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
     <AppendTargetFrameworkToOutputPath Condition="'$(BuildingInsideVisualStudio)' == 'true'">true</AppendTargetFrameworkToOutputPath>
-    <OutputPath>$(RuntimeBinDir)/ilc</OutputPath>
+    <OutputPath>$(RuntimeBinDir)ilc</OutputPath>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <EnableDefaultEmbeddedResourceItems>false</EnableDefaultEmbeddedResourceItems>
     <RuntimeIdentifiers>linux-x64;win-x64;osx-x64</RuntimeIdentifiers>
@@ -36,45 +36,53 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <ObjectWriterVersion>1.0.0-alpha-28820-01</ObjectWriterVersion>
+    <ILCompilerVersion>6.0.0-preview.1.21075.1</ILCompilerVersion>
     <CoreDisToolsVersion>1.0.1-prerelease-00005</CoreDisToolsVersion>
 
+    <ObjWriterArtifactPath>$(ArtifactsDir)llvm\build\bin\Release\$(LibraryNamePrefix)objwriter$(LibraryNameExtension)</ObjWriterArtifactPath>
+
     <!-- CoreDisTools are used in debugging visualizers. The package only exists on Windows -->
     <IncludeCoreDisTools Condition="'$(TargetsWindows)' == 'true' and '$(Configuration)' != 'Release'">true</IncludeCoreDisTools>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetArchitecture)' != 'arm64'">
-    <PackageReference Include="Microsoft.DotNet.ObjectWriter">
-      <Version>$(ObjectWriterVersion)</Version>
+    <PackageReference Condition="!Exists('$(ObjWriterArtifactPath)')" Include="Microsoft.DotNet.ILCompiler">
+      <Version>$(ILCompilerVersion)</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.NetCore.CoreDisTools" Condition="'$(IncludeCoreDisTools)' == 'true'">
       <Version>$(CoreDisToolsVersion)</Version>
     </PackageReference>
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetArchitecture)' != 'arm64'">
-    <Content Include="$(NuGetPackageRoot)\runtime.$(ILCompilerRuntimeIdentifier).microsoft.dotnet.objectwriter\$(ObjectWriterVersion)\runtimes\$(ILCompilerRuntimeIdentifier)\native\$(LibraryNamePrefix)objwriter$(LibraryNameExtension)">
+    <Content Condition="Exists('$(ObjWriterArtifactPath)')" Include="$(ObjWriterArtifactPath)">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <Visible>false</Visible>
+      <Pack>false</Pack>
+    </Content>
+
+    <Content Condition="!Exists('$(ObjWriterArtifactPath)')"
+      Include="$(NuGetPackageRoot)runtime.$(ILCompilerRuntimeIdentifier).microsoft.dotnet.ilcompiler\$(ILCompilerVersion)\tools\$(LibraryNamePrefix)objwriter$(LibraryNameExtension)">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
       <Pack>false</Pack>
     </Content>
 
-    <Content Include="$(NuGetPackageRoot)\runtime.$(ILCompilerRuntimeIdentifier).microsoft.netcore.coredistools\$(CoreDisToolsVersion)\runtimes\$(ILCompilerRuntimeIdentifier)\native\coredistools.dll" Condition="'$(IncludeCoreDisTools)' == 'true'">
+    <Content Include="$(NuGetPackageRoot)runtime.$(ILCompilerRuntimeIdentifier).microsoft.netcore.coredistools\$(CoreDisToolsVersion)\runtimes\$(ILCompilerRuntimeIdentifier)\native\coredistools.dll" Condition="'$(IncludeCoreDisTools)' == 'true'">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
       <Pack>false</Pack>
     </Content>
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="$(RuntimeBinDir)\$(LibraryNamePrefix)jitinterface_$(TargetArchitecture)$(LibraryNameExtension)">
+    <Content Include="$(RuntimeBinDir)$(LibraryNamePrefix)jitinterface_$(TargetArchitecture)$(LibraryNameExtension)">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
       <Pack>false</Pack>
     </Content>
 
-    <Content Include="$(RuntimeBinDir)\$(LibraryNamePrefix)clrjit$(LibraryNameExtension)">
-      <Link>$(LibraryNamePrefix)clrjitilc$(LibraryNameExtension)</Link>
+    <Content Include="$(RuntimeBinDir)$(LibraryNamePrefix)clrjit_*_$(TargetArchitecture)$(LibraryNameExtension)">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       <Visible>false</Visible>
       <Pack>false</Pack>
@@ -117,4 +125,21 @@
   <ItemGroup>
     <Compile Remove="repro\*" />
   </ItemGroup>
+
+  <PropertyGroup>
+    <TargetOSComponent>unix</TargetOSComponent>
+    <TargetOSComponent Condition="'$(TargetOS)' == 'windows'">win</TargetOSComponent>
+    <TargetSpec>$(TargetOSComponent)_$(TargetArchitecture)_$(TargetArchitecture)</TargetSpec>
+  </PropertyGroup>
+
+  <!-- On Windows we can re-use PGO-optimized clrjit.dll produced by the build for AOT compilation.
+       On Linux renaming the library makes it difficult to debug it. -->
+  <ItemGroup Condition="'$(TargetOS)' == 'windows'">
+    <Content Remove="$(RuntimeBinDir)$(LibraryNamePrefix)clrjit_$(TargetSpec)$(LibraryNameExtension)" />
+    <Content Include="$(RuntimeBinDir)$(LibraryNamePrefix)clrjit$(LibraryNameExtension)"
+      CopyToOutputDirectory="PreserveNewest"
+      CopyToPublishDirectory="PreserveNewest"
+      Link="$(LibraryNamePrefix)clrjit_$(TargetSpec)$(LibraryNameExtension)"
+      />
+  </ItemGroup>
 </Project>

src/coreclr/tools/aot/ObjWriter/build.cmd

@@ -13,11 +13,16 @@ git apply "%~dp0llvm.patch" || exit /b 1
 
 :: Copy ObjWriter files
 robocopy /mir "%~dp0\" tools\ObjWriter
-if %ErrorLevel% geq 8 exit /b %ErrorLevel%
+if %ErrorLevel% geq 8 exit /b 1
 
 :: Configure and build
 if not exist build mkdir build
 cd build || exit /b 1
 
-cmake ../ -DCMAKE_BUILD_TYPE=Release -DLLVM_OPTIMIZED_TABLEGEN=1 -DHAVE_POSIX_SPAWN=0 -DLLVM_ENABLE_PIC=1 -DLLVM_BUILD_TESTS=0 -DLLVM_ENABLE_DOXYGEN=0 -DLLVM_INCLUDE_DOCS=0 -DLLVM_INCLUDE_TESTS=0 -DLLVM_TARGETS_TO_BUILD="ARM;X86;AArch64" -DCMAKE_INSTALL_PREFIX=install || exit /b 1
-cmake --build . -j 10 --config Release --target install || exit /b 1
+:: Set CMakePath by evaluating the output from set-cmake-path.ps1
+call "%2src\coreclr\setup_vs_tools.cmd" || exit /b 1
+for /f "delims=" %%a in ('powershell -NoProfile -ExecutionPolicy ByPass "& ""%2eng\native\set-cmake-path.ps1"""') do %%a
+echo Using CMake at "%CMakePath%"
+
+"%CMakePath%" ../ -DCMAKE_BUILD_TYPE=Release -DLLVM_OPTIMIZED_TABLEGEN=1 -DHAVE_POSIX_SPAWN=0 -DLLVM_ENABLE_PIC=1 -DLLVM_BUILD_TESTS=0 -DLLVM_ENABLE_DOXYGEN=0 -DLLVM_INCLUDE_DOCS=0 -DLLVM_INCLUDE_TESTS=0 -DLLVM_TARGETS_TO_BUILD="ARM;X86;AArch64" -DCMAKE_INSTALL_PREFIX=install || exit /b 1
+"%CMakePath%" --build . -j 10 --config Release --target install || exit /b 1

src/coreclr/tools/aot/ObjWriter/build.sh

@@ -9,7 +9,7 @@ if [ ! -d llvm ]; then
 fi
 
 # Clean the tree and apply the patch
-cd llvm && git restore . || exit 1
+cd llvm && git checkout -- . || exit 1
 git apply "$ScriptPath/llvm.patch" || exit 1
 
 # Add ObjWriter files

src/coreclr/tools/aot/ObjWriter/objwriter.cpp

@@ -50,7 +50,7 @@
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/Win64EH.h"
 #include "llvm/Target/TargetMachine.h"
-#include "..\..\..\lib\Target\AArch64\MCTargetDesc\AArch64MCExpr.h"
+#include "../../../lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h"
 
 using namespace llvm;
 using namespace llvm::codeview;

src/coreclr/tools/aot/ObjWriter/objwriter.proj

@@ -8,7 +8,7 @@
 
   <Target Name="Build">
     <!-- Use IgnoreStandardErrorWarningFormat because Arcade sets WarnAsError and there are multiple warnings in the LLVM build -->
-    <Exec Command="$(BuildScriptName) $(ArtifactsDir)"
+    <Exec Command="&quot;$(MSBuildThisFileDirectory)$(BuildScriptName)&quot; $(ArtifactsDir) $(RepoRoot)"
           IgnoreStandardErrorWarningFormat="true" />
   </Target>
 </Project>