[build] enable One .NET MSBuild tests

This will enable all MSBuild tests in `Xamarin.Android.Build.Tests.dll` except for the new categories: `DotNetIgnore`, `AOT`, `FSharp`, `LibraryProjectZip`, `MkBundle`, `MonoSymbolicate`, `PackagesConfig`, and `StaticProject`. Reasoning: * `AOT` - need Mono AOT support in .NET 5. * `FSharp` - [Xamarin.Android.FSharp.ResourceProvider][0] support needed on .NET 5. * `LibraryProjectZip` - will need future support in Xamarin.Android. * `MkBundle` - may not ever come to .NET 5. * `MonoSymbolicate` - need symbolication support in .NET 5. * `PackagesConfig` - n/a for .NET 5. * `StaticProject` a few tests that were migrated to `Xamarin.Android.Build.Tests` in d447aa6 are ignored for now. This also reworks our build pipeline to run the MSBuild tests across 12 CI machines: * Windows - Node 1 - Legacy * Windows - Node 2 - Legacy * Windows - Node 3 - Legacy * Windows - Node 1 - One .NET * Windows - Node 2 - One .NET * Windows - Node 3 - One .NET * macOS - Node 1 - Legacy * macOS - Node 2 - Legacy * macOS - Node 3 - Legacy * macOS - Node 1 - One .NET * macOS - Node 2 - One .NET * macOS - Node 3 - One .NET Since this is a lot of machines: 1. I moved all the `One .NET` tests to their own phase. 2. The phase only runs when `RunAllTests=true`. This means the `One .NET` tests will run on master, release branches, and Mono bumps. You can manually queue a build to enable `RunAllTests` on a PR. To be completed in another PR, there are still more test assemblies that need to be run under a `dotnet` context: * `Xamarin.Android.Build.Tests.Commercial.dll` * `MSBuildDeviceIntegration.dll` [0]: https://github.com/xamarin/Xamarin.Android.FSharp.ResourceProvider
dotnet · Aug 6, 2020 · 015daae · 015daae
1 parent cde97d6
commit 015daae
Show file tree

Hide file tree

Showing 18 changed files with 347 additions and 188 deletions.
diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml
@@ -63,6 +63,7 @@ variables:
   # - This is a non-fork branch with name containing "mono-" (for Mono bumps)
   IsMonoBranch: $[and(eq(variables['XA.Commercial.Build'], 'true'), ne(variables['System.PullRequest.IsFork'], 'True'), or(contains(variables['Build.SourceBranchName'], 'mono-'), contains(variables['System.PullRequest.SourceBranch'], 'mono-')))]
   RunAllTests: $[or(eq(variables['XA.RunAllTests'], true), eq(variables['IsMonoBranch'], true))]
+  DotNetNUnitCategories: '&& cat != DotNetIgnore && cat != AOT && cat != FSharp && cat != LibraryProjectZip && cat != MkBundle && cat != MonoSymbolicate && cat != PackagesConfig && cat != StaticProject'
 
 # Stage and Job "display names" are shortened because they are combined to form the name of the corresponding GitHub check.
 stages:
@@ -796,31 +797,45 @@ stages:
 
     - template: yaml-templates/fail-on-issue.yaml
 
-  # Xamarin.Android (Test MSBuild - macOS)
+  # Xamarin.Android (Test MSBuild - macOS - Legacy)
   - template: yaml-templates\run-msbuild-mac-tests.yaml
     parameters:
       node_id: 1
+      job_name: mac_msbuild_tests_1
+      job_suffix: Legacy
 
   - template: yaml-templates\run-msbuild-mac-tests.yaml
     parameters:
       node_id: 2
+      job_name: mac_msbuild_tests_2
+      job_suffix: Legacy
+      run_extra_tests: true
 
   - template: yaml-templates\run-msbuild-mac-tests.yaml
     parameters:
       node_id: 3
+      job_name: mac_msbuild_tests_3
+      job_suffix: Legacy
 
-  # Xamarin.Android (Test MSBuild - Windows)
+  # Xamarin.Android (Test MSBuild - Windows - Legacy)
   - template: yaml-templates\run-msbuild-win-tests.yaml
     parameters:
       node_id: 1
+      job_name: win_msbuild_tests_1
+      job_suffix: Legacy
 
   - template: yaml-templates\run-msbuild-win-tests.yaml
     parameters:
       node_id: 2
+      job_name: win_msbuild_tests_2
+      job_suffix: Legacy
+      run_extra_tests: true
 
   - template: yaml-templates\run-msbuild-win-tests.yaml
     parameters:
       node_id: 3
+      job_name: win_msbuild_tests_3
+      job_suffix: Legacy
 
   # Check - "Xamarin.Android (Test MSBuild With Emulator - macOS)"
   - job: mac_msbuilddevice_tests
@@ -1026,6 +1041,62 @@ stages:
         targetPath: $(Build.ArtifactStagingDirectory)/designer-binlogs
       condition: always()
 
+- stage: dotnet_tests
+  displayName: One .NET Tests
+  dependsOn: mac_build
+  condition: and(eq(dependencies.mac_build.result, 'Succeeded'), eq(variables['RunAllTests'], true))
+  jobs:
+
+  # Xamarin.Android (Test MSBuild - macOS - One .NET)
+  - template: yaml-templates\run-msbuild-mac-tests.yaml
+    parameters:
+      node_id: 1
+      job_name: mac_dotnet_tests_1
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
+  - template: yaml-templates\run-msbuild-mac-tests.yaml
+    parameters:
+      node_id: 2
+      job_name: mac_dotnet_tests_2
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
+  - template: yaml-templates\run-msbuild-mac-tests.yaml
+    parameters:
+      node_id: 3
+      job_name: mac_dotnet_tests_3
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
+  # Xamarin.Android (Test MSBuild - Windows - One .NET)
+  - template: yaml-templates\run-msbuild-win-tests.yaml
+    parameters:
+      node_id: 1
+      job_name: win_dotnet_tests_1
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
+  - template: yaml-templates\run-msbuild-win-tests.yaml
+    parameters:
+      node_id: 2
+      job_name: win_dotnet_tests_2
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
+  - template: yaml-templates\run-msbuild-win-tests.yaml
+    parameters:
+      node_id: 3
+      job_name: win_dotnet_tests_3
+      job_suffix: One .NET
+      nunit_categories: $(DotNetNUnitCategories)
+      nunit_extra: --testparam dotnet=true
+
 - stage: integrated_regression_test
   displayName: Regression Tests
   dependsOn: mac_build

diff --git a/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml
@@ -2,10 +2,15 @@
 
 parameters:
   node_id: 0
+  job_name: ''
+  job_suffix: ''
+  nunit_categories: ''
+  nunit_extra: ''
+  run_extra_tests: false
 
 jobs:
-  - job: mac_msbuild_tests${{ parameters.node_id }}
-    displayName: MSBuild - Mac-${{ parameters.node_id }}
+  - job: ${{ parameters.job_name }}
+    displayName: MSBuild - Mac-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
     pool: $(HostedMac)
     timeoutInMinutes: 180
     cancelTimeoutInMinutes: 5
@@ -26,13 +31,13 @@ jobs:
 
     - template: run-nunit-tests.yaml
       parameters:
-        testRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }}
+        testRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
-        nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }}"
-        testResultsFile: TestResult-MSBuildTests-macOS-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
+        nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}" ${{ parameters.nunit_extra }}
+        testResultsFile: TestResult-MSBuildTests-${{ parameters.job_name }}-$(XA.Build.Configuration).xml
 
     # Only run these tests on node 2
-    - ${{ if eq(parameters.node_id, 2) }}:
+    - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
           testRunTitle: Xamarin.Android.Build.Tests.Commercial - macOS
@@ -46,15 +51,8 @@ jobs:
           nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
           testResultsFile: TestResult-MSBuildTests-macOS-NoNode-$(XA.Build.Configuration).xml
 
-      - template: run-nunit-tests.yaml
-        parameters:
-          testRunTitle: Xamarin.Android.Build.Tests - macOS - One .NET
-          testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
-          nunitConsoleExtraArgs: --where "cat == dotnet" --params dotnet=true
-          testResultsFile: TestResult-MSBuildTests-macOS-dotnet-$(XA.Build.Configuration).xml
-
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - MSBuild - Mac-${{ parameters.node_id }}
+        artifactName: Test Results - MSBuild - Mac-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
 
     - template: fail-on-issue.yaml
diff --git a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml
@@ -2,10 +2,15 @@
 
 parameters:
   node_id: 0
+  job_name: ''
+  job_suffix: ''
+  nunit_categories: ''
+  nunit_extra: ''
+  run_extra_tests: false
 
 jobs:
-  - job: win_msbuild_tests${{ parameters.node_id }}
-    displayName: MSBuild - Windows-${{ parameters.node_id }}
+  - job: ${{ parameters.job_name }}
+    displayName: MSBuild - Windows-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
     pool: $(VSEngWinVS2019)
     timeoutInMinutes: 180
     cancelTimeoutInMinutes: 5
@@ -37,13 +42,13 @@ jobs:
     # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically.
     - template: run-nunit-tests.yaml
       parameters:
-        testRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }}
+        testRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
-        nunitConsoleExtraArgs: --workers=4 --where "cat == Node-${{ parameters.node_id }}"
-        testResultsFile: TestResult-MSBuildTests-Windows-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
+        nunitConsoleExtraArgs: --workers=4 --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}" ${{ parameters.nunit_extra }}
+        testResultsFile: TestResult-MSBuildTests-${{ parameters.job_name }}-$(XA.Build.Configuration).xml
 
     # Only run these tests on node 2
-    - ${{ if eq(parameters.node_id, 2) }}:
+    - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
           testRunTitle: Xamarin.Android.Build.Tests.Commercial - Windows
@@ -55,17 +60,10 @@ jobs:
           testRunTitle: Xamarin.Android.Build.Tests - Windows - No Node
           testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
           nunitConsoleExtraArgs: --workers=4 --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
-          testResultsFile: TestResult-MSBuildTests-Windows-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
-
-      - template: run-nunit-tests.yaml
-        parameters:
-          testRunTitle: Xamarin.Android.Build.Tests - Windows - One .NET
-          testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
-          nunitConsoleExtraArgs: --workers=4 --where "cat == dotnet" --params dotnet=true
-          testResultsFile: TestResult-MSBuildTests-Windows-dotnet-$(XA.Build.Configuration).xml
+          testResultsFile: TestResult-MSBuildTests-Windows-NoNode-$(XA.Build.Configuration).xml
 
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }}
+        artifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
 
     - template: fail-on-issue.yaml
diff --git a/...marin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs b/...marin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs
@@ -17,17 +17,6 @@ namespace Xamarin.Android.Build.Tests
 	[Parallelizable (ParallelScope.Children)]
 	public class AndroidUpdateResourcesTest : BaseTest
 	{
-		string GetResourceDesignerPath (ProjectBuilder builder, XamarinAndroidProject project)
-		{
-			string path;
-			if (Builder.UseDotNet) {
-				path = Path.Combine (Root, builder.ProjectDirectory, project.IntermediateOutputPath);
-			} else {
-				path = Path.Combine (Root, builder.ProjectDirectory, "Resources");
-			}
-			return Path.Combine (path, "Resource.designer" + project.Language.DefaultDesignerExtension);
-		}
-
 		[Test]
 		[Category ("dotnet")]
 		public void CheckMultipleLibraryProjectReferenceAlias ([Values (true, false)] bool withGlobal)
@@ -183,9 +172,11 @@ public void CheckEmbeddedSupportLibraryResources ()
 					KnownPackages.AndroidSupportV4_27_0_2_1,
 					KnownPackages.SupportV7AppCompat_27_0_2_1,
 				},
-				TargetFrameworkVersion = "v7.1",
 			};
-			using (var b = CreateApkBuilder ("temp/CheckEmbeddedSupportLibraryResources")) {
+			if (Builder.UseDotNet) {
+				proj.AddDotNetCompatPackages ();
+			}
+			using (var b = CreateApkBuilder ()) {
 				Assert.IsTrue (b.Build (proj), "First build should have succeeded.");
 				var Rdrawable = b.Output.GetIntermediaryPath (Path.Combine ("android", "bin", "classes", "android", "support", "v7", "appcompat", "R$drawable.class"));
 				Assert.IsTrue (File.Exists (Rdrawable), $"{Rdrawable} should exist");
@@ -292,6 +283,7 @@ public void RepetiviteBuildUpdateSingleResource ([Values (false, true)] bool use
 
 		[Test]
 		[NonParallelizable]
+		[Category ("DotNetIgnore")] // <ProcessGoogleServicesJson/> task is built for net45
 		public void Check9PatchFilesAreProcessed ()
 		{
 			var projectPath = Path.Combine ("temp", "Check9PatchFilesAreProcessed");
@@ -561,6 +553,9 @@ public void CheckResourceDesignerIsCreated (bool isRelease, ProjectLanguage lang
 			if (IsWindows && isFSharp) {
 				Assert.Ignore ("Skipping this F# test on Windows.");
 			}
+			if (Builder.UseDotNet && isFSharp) {
+				Assert.Ignore ("Skipping this F# test under 'dotnet'.");
+			}
 
 			var proj = new XamarinAndroidApplicationProject () {
 				Language = language,
@@ -589,17 +584,21 @@ public void CheckResourceDesignerIsUpdatedWhenReadOnly (bool isRelease, ProjectL
 		{
 			//Due to the MSBuild project automatically sorting <ItemGroup />, we can't possibly get the F# projects to build here on Windows
 			//  This API is sorting them: https://github.com/xamarin/xamarin-android/blob/c588bfe07aab224c97996a264579f4d4f18a151c/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetXamarinProject.cs#L117
-			if (IsWindows && language == XamarinAndroidProjectLanguage.FSharp) {
+			bool isFSharp = language == XamarinAndroidProjectLanguage.FSharp;
+			if (IsWindows && isFSharp) {
 				Assert.Ignore ("Skipping this F# test on Windows.");
 			}
+			if (Builder.UseDotNet && isFSharp) {
+				Assert.Ignore ("Skipping this F# test under 'dotnet'.");
+			}
 
 			var proj = new XamarinAndroidApplicationProject () {
 				Language = language,
 				IsRelease = isRelease,
 			};
 			using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) {
 				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
-				var designerPath = Path.Combine (Root, b.ProjectDirectory, "Resources", "Resource.designer" + proj.Language.DefaultDesignerExtension);
+				var designerPath = GetResourceDesignerPath (b, proj);
 				var attr = File.GetAttributes (designerPath);
 				File.SetAttributes (designerPath, FileAttributes.ReadOnly);
 				Assert.IsTrue ((File.GetAttributes (designerPath) & FileAttributes.ReadOnly) == FileAttributes.ReadOnly,
@@ -629,13 +628,9 @@ public void CheckResourceDesignerIsUpdatedWhenReadOnly (bool isRelease, ProjectL
 		}
 
 		[Test]
-		[TestCaseSource(nameof (ReleaseLanguage))]
-		public void CheckOldResourceDesignerIsNotUsed (bool isRelease, ProjectLanguage language)
+		public void CheckOldResourceDesignerIsNotUsed ([Values (true, false)] bool isRelease)
 		{
-			if (language == XamarinAndroidProjectLanguage.FSharp)
-				Assert.Ignore ("Skipping CheckOldResourceDesignerIsNotUsed for FSharp until Xamarin.Android.FSharp.ResourceProvider supports it.");
 			var proj = new XamarinAndroidApplicationProject () {
-				Language = language,
 				IsRelease = isRelease,
 			};
 			proj.SetProperty ("AndroidUseIntermediateDesignerFile", "True");
@@ -660,13 +655,9 @@ public void CheckOldResourceDesignerIsNotUsed (bool isRelease, ProjectLanguage l
 
 		// ref https://bugzilla.xamarin.com/show_bug.cgi?id=30089
 		[Test]
-		[TestCaseSource(nameof (ReleaseLanguage))]
-		public void CheckOldResourceDesignerWithWrongCasingIsRemoved (bool isRelease, ProjectLanguage language)
+		public void CheckOldResourceDesignerWithWrongCasingIsRemoved ([Values (true, false)] bool isRelease)
 		{
-			if (language == XamarinAndroidProjectLanguage.FSharp)
-				Assert.Ignore ("Skipping CheckOldResourceDesignerIsNotUsed for FSharp until Xamarin.Android.FSharp.ResourceProvider supports it.");
 			var proj = new XamarinAndroidApplicationProject () {
-				Language = language,
 				IsRelease = isRelease,
 			};
 			proj.SetProperty ("AndroidUseIntermediateDesignerFile", "True");
@@ -1100,6 +1091,7 @@ public void BuildAppWithManagedResourceParserAndLibraries ()
 		}
 
 		[Test]
+		[Category ("DotNetIgnore")] // n/a in .NET 5, not possible to use $(TFV) of v8.0
 		public void CheckMaxResWarningIsEmittedAsAWarning([Values (false, true)] bool useAapt2)
 		{
 			var path = Path.Combine ("temp", TestName);
@@ -1301,6 +1293,7 @@ public void CustomViewAddResourceId ([Values (false, true)] bool useAapt2)
 		// https://github.com/xamarin/xamarin-android/issues/2205
 		[Test]
 		[NonParallelizable]
+		[Category ("DotNetIgnore")] // <ProcessGoogleServicesJson/> task is built for net45
 		public void Issue2205 ([Values (false, true)] bool useAapt2)
 		{
 			var proj = new XamarinAndroidApplicationProject ();