[release/6.0-preview7] [wasm] Add support for using custom native lib…

…raries (#56013)

* [wasm] Add support for using custom native libraries (#55797)

(cherry picked from commit d574b03)

* [wasm] Use compile rsp instead of link, for compiling native files (#55848)

.. and fix logging that broke recently.

`tasks/Common/Utils.cs`:

TaskLoggingHelper Utils.Logger is a static field, which must be set by
task else any methods in Utils, eg. RunProcess, silently fail to log
any messages. Also, this would be a problem when building multiple
projects in parallel, since the logger is a task-specific one.

Instead, we pass logger as an arg to all the methods.

(cherry picked from commit 3301e9d)

* Link with EmccCompileOptimizationFlag==-Oz by default in release (#55939)

(cherry picked from commit 04072ff)

* [wasm] Fix regression in compiling .bc -> .o files (#56063)

* [wasm] Add back --emit-llvm that got removed mistakenly, in an earlier commit

.. found thanks to Jerome Laban.

* [wasm] Set EmccCompile's messages to MessageImportance.Low by default.

.. and to MessageImportance.Normal if `$(EmccVerbose)==true`.

* [wasm] Quote filenames passed to emcc compile command line

* Add more blazorwasm tests - for debug/release, aot/relinking

* Bump sdk for workload testing to 6.0.100-rc.1.21370.2

* [wasm] Fix regression in compiling bitcode -> .o

The `-emit-llvm` arg has been incorrectly added, and removed from the
args used for compiling .bc->.o .

This commit fixes it, and adds a crude test for it, so we don't regress
again.

* Fix build

(cherry picked from commit 1d8ad03)

* [wasm] Bump sdk for workload testing to 6.0.100-preview.7.21372.19

Co-authored-by: Larry Ewing <lewing@microsoft.com>

eng/Versions.props

@@ -162,7 +162,7 @@
     <SQLitePCLRawbundle_greenVersion>2.0.4</SQLitePCLRawbundle_greenVersion>
     <MoqVersion>4.12.0</MoqVersion>
     <FsCheckVersion>2.14.3</FsCheckVersion>
-    <SdkVersionForWorkloadTesting>6.0.100-preview.7.21362.5</SdkVersionForWorkloadTesting>
+    <SdkVersionForWorkloadTesting>6.0.100-preview.7.21372.19</SdkVersionForWorkloadTesting>
     <!-- Docs -->
     <MicrosoftPrivateIntellisenseVersion>5.0.0-preview-20201009.2</MicrosoftPrivateIntellisenseVersion>
     <!-- ILLink -->

src/mono/wasm/build/WasmApp.Native.targets

@@ -18,6 +18,7 @@
     </_WasmBuildNativeCoreDependsOn>
 
     <WasmBuildNativeOnlyDependsOn>
+      _InitializeCommonProperties;
       _PrepareForWasmBuildNativeOnly;
       _WasmBuildNativeCore;
     </WasmBuildNativeOnlyDependsOn>
@@ -38,13 +39,14 @@
   <Target Name="WasmBuildNativeOnly" DependsOnTargets="$(WasmBuildNativeOnlyDependsOn)" Condition="'$(WasmBuildNative)' == 'true'" />
 
   <Target Name="_PrepareForWasmBuildNativeOnly">
+    <MakeDir Directories="$(_WasmIntermediateOutputPath)" />
+
     <ItemGroup>
       <_WasmAssembliesInternal Remove="@(_WasmAssembliesInternal)" />
       <_WasmAssembliesInternal Include="@(WasmAssembliesToBundle->Distinct())" />
     </ItemGroup>
   </Target>
 
-
   <Target Name="_SetupEmscripten">
     <PropertyGroup>
       <_EMSDKMissingPaths Condition="'$(_EMSDKMissingPaths)' == '' and ('$(EmscriptenSdkToolsPath)' == '' or !Exists('$(EmscriptenSdkToolsPath)'))">%24(EmscriptenSdkToolsPath)=$(EmscriptenSdkToolsPath) </_EMSDKMissingPaths>
@@ -115,8 +117,9 @@
 
     <PropertyGroup>
       <WasmBuildNative Condition="'$(RunAOTCompilation)' == 'true'">true</WasmBuildNative>
-      <WasmBuildNative Condition="'$(WasmBuildNative)' == '' and '$(PublishTrimmed)' != 'true'">false</WasmBuildNative>
-      <WasmBuildNative Condition="'$(WasmBuildNative)' == '' and '$(Configuration)' == 'Release'">true</WasmBuildNative>
+      <WasmBuildNative Condition="'$(WasmBuildNative)' == '' and @(NativeFileReference->Count()) > 0" >true</WasmBuildNative>
+      <WasmBuildNative Condition="'$(WasmBuildNative)' == '' and '$(PublishTrimmed)' != 'true'"   >false</WasmBuildNative>
+      <WasmBuildNative Condition="'$(WasmBuildNative)' == '' and '$(Configuration)' == 'Release'" >true</WasmBuildNative>
       <WasmBuildNative Condition="'$(WasmBuildNative)' == ''">false</WasmBuildNative>
     </PropertyGroup>
 
@@ -152,7 +155,11 @@
       <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz</_EmccOptimizationFlagDefault>
 
       <EmccCompileOptimizationFlag Condition="'$(EmccCompileOptimizationFlag)' == ''">$(_EmccOptimizationFlagDefault)</EmccCompileOptimizationFlag>
-      <EmccLinkOptimizationFlag    Condition="'$(EmccLinkOptimizationFlag)' == ''"   >-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault)</EmccLinkOptimizationFlag>
+      <EmccLinkOptimizationFlag    Condition="'$(EmccLinkOptimizationFlag)' == ''"   >$(EmccCompileOptimizationFlag)</EmccLinkOptimizationFlag>
+
+      <_EmccCompileRsp>$(_WasmIntermediateOutputPath)emcc-compile.rsp</_EmccCompileRsp>
+      <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' == 'true'">Normal</_EmccCompileOutputMessageImportance>
+      <_EmccCompileOutputMessageImportance Condition="'$(EmccVerbose)' != 'true'">Low</_EmccCompileOutputMessageImportance>
     </PropertyGroup>
 
     <ItemGroup>
@@ -161,15 +168,42 @@
       <_EmccCommonFlags Include="-s DISABLE_EXCEPTION_CATCHING=0" />
       <_EmccCommonFlags Include="-g"                                Condition="'$(WasmNativeStrip)' == 'false'" />
       <_EmccCommonFlags Include="-v"                                Condition="'$(EmccVerbose)' != 'false'" />
+
+      <_EmccIncludePaths Include="$(_WasmIntermediateOutputPath.TrimEnd('\/'))" />
+      <_EmccIncludePaths Include="$(_WasmRuntimePackIncludeDir)mono-2.0" />
+      <_EmccIncludePaths Include="$(_WasmRuntimePackIncludeDir)wasm" />
+
+      <!-- Adding optimization flag at the top, so it gets precedence -->
+      <_EmccCFlags Include="$(EmccCompileOptimizationFlag)" />
+      <_EmccCFlags Include="@(_EmccCommonFlags)" />
+
+      <_EmccCFlags Include="-DENABLE_AOT=1"                    Condition="'$(RunAOTCompilation)' == 'true'" />
+      <_EmccCFlags Include="-DDRIVER_GEN=1"                    Condition="'$(RunAOTCompilation)' == 'true'" />
+      <_EmccCFlags Include="-DINVARIANT_GLOBALIZATION=1"       Condition="'$(InvariantGlobalization)' == 'true'" />
+      <_EmccCFlags Include="-DLINK_ICALLS=1"                   Condition="'$(WasmLinkIcalls)' == 'true'" />
+      <_EmccCFlags Include="-DCORE_BINDINGS" />
+      <_EmccCFlags Include="-DGEN_PINVOKE=1" />
+      <_EmccCFlags Include="-emit-llvm" />
+
+      <_EmccCFlags Include="&quot;-I%(_EmccIncludePaths.Identity)&quot;" />
+      <_EmccCFlags Include="-g" Condition="'$(WasmNativeDebugSymbols)' == 'true'" />
+
+      <_EmccCFlags Include="$(EmccExtraCFlags)" />
+
+      <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)*.c" />
+      <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" />
+
+      <_DotnetJSSrcFile Include="$(_WasmRuntimePackSrcDir)\*.js" />
+      <_WasmNativeFileForLinking Include="@(NativeFileReference)" />
     </ItemGroup>
 
-   <ItemGroup>
-     <_DotnetJSSrcFile Include="$(_WasmRuntimePackSrcDir)\*.js" />
-   </ItemGroup>
+    <Error Text="Could not find NativeFileReference %(NativeFileReference.Identity)" Condition="'%(NativeFileReference.Identity)' != '' and !Exists(%(NativeFileReference.Identity))" />
   </Target>
 
   <Target Name="_GeneratePInvokeTable">
     <ItemGroup>
+      <_WasmPInvokeModules Include="%(_WasmNativeFileForLinking.FileName)" Condition="'%(_WasmNativeFileForLinking.ScanForPInvokes)' != 'false'" />
+
       <_WasmPInvokeModules Include="libSystem.Native" />
       <_WasmPInvokeModules Include="libSystem.IO.Compression.Native" />
       <_WasmPInvokeModules Include="libSystem.Globalization.Native" />
@@ -194,38 +228,13 @@
 
   <Target Name="_WasmCompileNativeFiles">
     <ItemGroup>
-      <_EmccIncludePaths Include="$(_WasmIntermediateOutputPath.TrimEnd('\/'))" />
-      <_EmccIncludePaths Include="$(_WasmRuntimePackIncludeDir)mono-2.0" />
-      <_EmccIncludePaths Include="$(_WasmRuntimePackIncludeDir)wasm" />
-
-      <!-- Adding optimization flag at the top, so it gets precedence -->
-      <_EmccCFlags Include="$(EmccCompileOptimizationFlag)" />
-      <_EmccCFlags Include="@(_EmccCommonFlags)" />
-
-      <_EmccCFlags Include="-DENABLE_AOT=1"                    Condition="'$(RunAOTCompilation)' == 'true'" />
-      <_EmccCFlags Include="-DDRIVER_GEN=1"                    Condition="'$(RunAOTCompilation)' == 'true'" />
-      <_EmccCFlags Include="-DINVARIANT_GLOBALIZATION=1"       Condition="'$(InvariantGlobalization)' == 'true'" />
-      <_EmccCFlags Include="-DLINK_ICALLS=1"                   Condition="'$(WasmLinkIcalls)' == 'true'" />
-      <_EmccCFlags Include="-DCORE_BINDINGS" />
-      <_EmccCFlags Include="-DGEN_PINVOKE=1" />
-      <_EmccCFlags Include="-emit-llvm" />
-
-      <_EmccCFlags Include="&quot;-I%(_EmccIncludePaths.Identity)&quot;" />
-      <_EmccCFlags Include="-g" Condition="'$(WasmNativeDebugSymbols)' == 'true'" />
-      <_EmccCFlags Include="-s EXPORTED_FUNCTIONS='[@(_ExportedFunctions->'&quot;%(Identity)&quot;', ',')]'" Condition="@(_ExportedFunctions->Count()) > 0" />
-
-      <_EmccCFlags Include="$(EmccExtraCFlags)" />
-
-      <_WasmRuntimePackSrcFile Remove="@(_WasmRuntimePackSrcFile)" />
-      <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)\*.c" />
-      <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" />
+      <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" />
       <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" />
     </ItemGroup>
 
     <PropertyGroup>
       <_EmBuilder Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.bat</_EmBuilder>
       <_EmBuilder Condition="!$([MSBuild]::IsOSPlatform('WINDOWS'))">embuilder.py</_EmBuilder>
-      <_EmccCompileRsp>$(_WasmIntermediateOutputPath)emcc-compile.rsp</_EmccCompileRsp>
     </PropertyGroup>
 
     <WriteLinesToFile Lines="@(_EmccCFlags)" File="$(_EmccCompileRsp)" Overwrite="true" WriteOnlyWhenDifferent="true" />
@@ -234,7 +243,15 @@
     <Exec Command="$(_EmBuilder) build MINIMAL" EnvironmentVariables="@(EmscriptenEnvVars)" StandardOutputImportance="Low" StandardErrorImportance="Low" />
 
     <Message Text="Compiling native assets with emcc. This may take a while ..." Importance="High" />
-    <EmccCompile SourceFiles="@(_WasmSourceFileToCompile)" Arguments='"@$(_EmccDefaultFlagsRsp)" "@$(_EmccCompileRsp)"' EnvironmentVariables="@(EmscriptenEnvVars)" />
+    <EmccCompile
+          SourceFiles="@(_WasmSourceFileToCompile)"
+          Arguments='"@$(_EmccDefaultFlagsRsp)" "@$(_EmccCompileRsp)"'
+          EnvironmentVariables="@(EmscriptenEnvVars)"
+          OutputMessageImportance="$(_EmccCompileOutputMessageImportance)" />
+
+    <ItemGroup>
+      <WasmNativeAsset Include="%(_WasmSourceFileToCompile.ObjectFile)" />
+    </ItemGroup>
   </Target>
 
   <ItemGroup Condition="'$(Configuration)' == 'Debug' and '@(_MonoComponent->Count())' == 0">
@@ -259,8 +276,9 @@
     <EmccCompile
           Condition="@(_BitCodeFile->Count()) > 0"
           SourceFiles="@(_BitCodeFile)"
-          Arguments="&quot;@$(_EmccDefaultFlagsRsp)&quot; @(_EmccLDFlags->'%(Identity)', ' ')"
-          EnvironmentVariables="@(EmscriptenEnvVars)" />
+          Arguments="&quot;@$(_EmccDefaultFlagsRsp)&quot; @(_EmccLDFlags, ' ')"
+          EnvironmentVariables="@(EmscriptenEnvVars)"
+          OutputMessageImportance="$(_EmccCompileOutputMessageImportance)" />
 
     <ItemGroup>
       <!-- order seems to matter -->
@@ -271,6 +289,8 @@
           Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)\*.a"
           Exclude="@(_MonoRuntimeComponentDontLink->'$(MicrosoftNetCoreAppRuntimePackRidNativeDir)\%(Identity)')" />
 
+      <_WasmExtraJSFile Include="@(Content)" Condition="'%(Content.Extension)' == '.js'" />
+
       <_EmccLinkStepArgs Include="@(_EmccLDFlags)" />
       <_EmccLinkStepArgs Include="--js-library &quot;%(_DotnetJSSrcFile.Identity)&quot;" />
       <_EmccLinkStepArgs Include="--js-library &quot;%(_WasmExtraJSFile.Identity)&quot;" Condition="'%(_WasmExtraJSFile.Kind)' == 'js-library'" />
@@ -483,6 +503,5 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_
     <ParameterGroup>
       <EmccProperties ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="false" Output="true" />
     </ParameterGroup>
- </UsingTask>
-
+  </UsingTask>
 </Project>

src/mono/wasm/build/WasmApp.props

@@ -7,6 +7,7 @@
 
     <WasmBuildAppAfterThisTarget Condition="'$(WasmBuildAppAfterThisTarget)' == ''">Publish</WasmBuildAppAfterThisTarget>
     <WasmBuildAppDependsOn>
+        _InitializeCommonProperties;
         _BeforeWasmBuildApp;
         _WasmResolveReferences;
         _WasmAotCompileApp;

src/mono/wasm/build/WasmApp.targets

@@ -80,6 +80,9 @@
     <!--<WasmStripAOTAssemblies Condition="'$(AOTMode)' == 'LLVMOnlyInterp'">false</WasmStripAOTAssemblies>-->
     <!--<WasmStripAOTAssemblies Condition="'$(WasmStripAOTAssemblies)' == ''">$(RunAOTCompilation)</WasmStripAOTAssemblies>-->
     <WasmStripAOTAssemblies>false</WasmStripAOTAssemblies>
+
+    <!-- emcc, and mono-aot-cross don't like relative paths for output files -->
+    <_WasmIntermediateOutputPath>$([MSBuild]::NormalizeDirectory($(IntermediateOutputPath), 'wasm'))</_WasmIntermediateOutputPath>
     <_BeforeWasmBuildAppDependsOn />
   </PropertyGroup>
 
@@ -90,7 +93,7 @@
 
   <Target Name="_WasmCoreBuild" BeforeTargets="WasmBuildApp" DependsOnTargets="$(WasmBuildAppDependsOn)" />
 
-  <Target Name="_BeforeWasmBuildApp" DependsOnTargets="$(_BeforeWasmBuildAppDependsOn)">
+  <Target Name="_InitializeCommonProperties">
     <Error Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == '' and ('%(ResolvedRuntimePack.PackageDirectory)' == '' or !Exists(%(ResolvedRuntimePack.PackageDirectory)))"
            Text="Could not find %25(ResolvedRuntimePack.PackageDirectory)=%(ResolvedRuntimePack.PackageDirectory)" />
 
@@ -100,11 +103,12 @@
       <MicrosoftNetCoreAppRuntimePackRidDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidDir)))</MicrosoftNetCoreAppRuntimePackRidDir>
       <MicrosoftNetCoreAppRuntimePackRidNativeDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidDir), 'native'))</MicrosoftNetCoreAppRuntimePackRidNativeDir>
 
-      <!-- FIXME: confirm that this won't get used before this -->
       <_WasmRuntimePackIncludeDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidNativeDir), 'include'))</_WasmRuntimePackIncludeDir>
       <_WasmRuntimePackSrcDir>$([MSBuild]::NormalizeDirectory($(MicrosoftNetCoreAppRuntimePackRidNativeDir), 'src'))</_WasmRuntimePackSrcDir>
     </PropertyGroup>
+  </Target>
 
+  <Target Name="_BeforeWasmBuildApp" DependsOnTargets="$(_BeforeWasmBuildAppDependsOn)">
     <Error Condition="'$(IntermediateOutputPath)' == ''" Text="%24(IntermediateOutputPath) property needs to be set" />
     <Error Condition="!Exists('$(MicrosoftNetCoreAppRuntimePackRidDir)')" Text="MicrosoftNetCoreAppRuntimePackRidDir=$(MicrosoftNetCoreAppRuntimePackRidDir) doesn't exist" />
     <Error Condition="@(WasmAssembliesToBundle->Count()) == 0" Text="WasmAssembliesToBundle item is empty. No assemblies to process" />
@@ -115,8 +119,6 @@
       <WasmMainAssemblyFileName Condition="'$(WasmMainAssemblyFileName)' == ''">$(TargetFileName)</WasmMainAssemblyFileName>
 
       <WasmAppDir>$([MSBuild]::NormalizeDirectory($(WasmAppDir)))</WasmAppDir>
-      <!-- emcc, and mono-aot-cross don't like relative paths for output files -->
-      <_WasmIntermediateOutputPath>$([MSBuild]::NormalizeDirectory($(IntermediateOutputPath), 'wasm'))</_WasmIntermediateOutputPath>
     </PropertyGroup>
 
     <MakeDir Directories="$(_WasmIntermediateOutputPath)" />

src/tasks/AndroidAppBuilder/AndroidApkFileReplacerTask.cs

@@ -26,8 +26,7 @@ public class AndroidApkFileReplacerTask : Task
 
     public override bool Execute()
     {
-        Utils.Logger = Log;
-        var apkBuilder = new ApkBuilder();
+        var apkBuilder = new ApkBuilder(Log);
         apkBuilder.OutputDir = OutputDir;
         apkBuilder.AndroidSdk = AndroidSdk;
         apkBuilder.MinApiLevel = MinApiLevel;

src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs

@@ -87,11 +87,9 @@ public class AndroidAppBuilderTask : Task
 
     public override bool Execute()
     {
-        Utils.Logger = Log;
-
         string abi = DetermineAbi();
 
-        var apkBuilder = new ApkBuilder();
+        var apkBuilder = new ApkBuilder(Log);
         apkBuilder.ProjectName = ProjectName;
         apkBuilder.AppDir = AppDir;
         apkBuilder.OutputDir = OutputDir;