[ci] Use AZDO built-in parallelization strategy.

build-tools/automation/azure-pipelines.yaml

@@ -652,116 +652,12 @@ stages:
       nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
       target_framework: $(DotNetStableTargetFramework)
       provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-- stage: msbuilddevice_tests
-  displayName: MSBuild Emulator Tests
-  dependsOn: mac_build
-  condition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuildDevice')))
-  jobs:
-  # Check - "Xamarin.Android (macOS > Tests > MSBuild+Emulator One .NET #N)"
-  - template: yaml-templates/run-msbuild-device-tests.yaml
-    parameters:
-      node_id: 1
-      job_name: mac_dotnetdevice_tests_1
-      job_suffix: One .NET
-      nunit_categories: $(DotNetNUnitCategories)
-      target_framework: $(DotNetStableTargetFramework)
-      provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-  - template: yaml-templates/run-msbuild-device-tests.yaml
-    parameters:
-      node_id: 2
-      job_name: mac_dotnetdevice_tests_2
-      job_suffix: One .NET
-      nunit_categories: $(DotNetNUnitCategories)
-      target_framework: $(DotNetStableTargetFramework)
-      provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-  - template: yaml-templates/run-msbuild-device-tests.yaml
-    parameters:
-      node_id: 3
-      job_name: mac_dotnetdevice_tests_3
-      job_suffix: One .NET
-      nunit_categories: $(DotNetNUnitCategories)
-      target_framework: $(DotNetStableTargetFramework)
-      provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-  - template: yaml-templates/run-msbuild-device-tests.yaml
-    parameters:
-      node_id: 4
-      job_name: mac_dotnetdevice_tests_4
-      job_suffix: One .NET
-      nunit_categories: $(DotNetNUnitCategories)
-      target_framework: $(DotNetStableTargetFramework)
-      provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-  - job: wear_tests
-    displayName: macOS > Tests > WearOS 
-    timeoutInMinutes: 180
-    cancelTimeoutInMinutes: 2
-    strategy:
-      matrix:
-        Android30-x86:
-          avdApiLevel: 30
-          avdAbi: x86
-          avdType: android-wear
-          deviceName: wear_square
-    pool:
-      vmImage: $(HostedMacImage)
-    workspace:
-      clean: all
-    steps:
-    - template: yaml-templates/setup-test-environment.yaml
-      parameters:
-        configuration: $(XA.Build.Configuration)
-
-    - template: yaml-templates/run-xaprepare.yaml
-      parameters:
-        displayName: install required brew tools and prepare java.interop
-        arguments: --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes
-
-    - template: yaml-templates/run-xaprepare.yaml
-      parameters:
-        displayName: install emulator
-        arguments: --s=EmulatorTestDependencies
-
-    - script: echo "##vso[task.setvariable variable=Java8SdkDirectory]$JAVA_HOME_8_X64"
-      displayName: set Java8SdkDirectory
-
-    - task: DownloadPipelineArtifact@2
-      inputs:
-        artifactName: $(TestAssembliesArtifactName)
-        downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
-
-    - task: MSBuild@1
-      displayName: install and launch emulator
-      inputs:
-        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
-        configuration: $(XA.Build.Configuration)
-        msbuildArguments: /t:InstallAvdImage;AcquireAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-$(avdApiLevel).binlog
-
-    - template: yaml-templates/run-nunit-tests.yaml
-      parameters:
-        testRunTitle: WearOS On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
-        dotNetTestExtraArgs: --filter "TestCategory = WearOS"
-        testResultsFile: TestResult-WearOS--$(XA.Build.Configuration).xml
-
-    - task: MSBuild@1
-      displayName: shut down emulator
-      inputs:
-        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
-        configuration: $(XA.Build.Configuration)
-        msbuildArguments: /t:AcquireAndroidTarget,ReleaseAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
-      condition: always()
-
-    - template: yaml-templates/upload-results.yaml
-      parameters:
-        configuration: $(XA.Build.Configuration)
-        artifactName: Test Results - Emulator $(avdApiLevel)-$(avdAbi)-$(avdType) - macOS
-
-    - template: yaml-templates/fail-on-issue.yaml
-
+
+- template: yaml-templates/stage-msbuild-emulator-tests.yaml
+  parameters:
+    provisionatorChannel: ${{ parameters.provisionatorChannel }}
+    stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuildDevice')))
+    nunit_categories: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != PackagesConfig & cat != StaticProject & cat != SystemApplication'
 
 - stage: bcl_tests
   displayName: BCL Emulator Tests

build-tools/automation/yaml-templates/install-dotnet-test-slicer.yaml

@@ -0,0 +1,15 @@
+parameters:
+  version: '0.1.0-alpha1'
+  condition: succeeded()
+  continueOnError: true
+
+steps:
+- powershell: dotnet tool uninstall dotnet-test-slicer -g
+  displayName: uninstall dotnet-test-slicer
+  ignoreLASTEXITCODE: true
+  condition: ${{ parameters.condition }}
+
+- script: dotnet tool update dotnet-test-slicer --version ${{ parameters.version }} --add-source https://api.nuget.org/v3/index.json -g
+  displayName: install dotnet-test-slicer ${{ parameters.version }}
+  condition: ${{ parameters.condition }}
+  continueOnError: ${{ parameters.continueOnError }}

build-tools/automation/yaml-templates/setup-test-environment.yaml

@@ -7,6 +7,7 @@ parameters:
   updateVS: false
   jdkTestFolder: $(JAVA_HOME_11_X64)
   remove_dotnet: false
+  installTestSlicer: false
 
 steps:
 - checkout: self
@@ -101,3 +102,6 @@ steps:
     arguments: -t:ExtractWorkloadPacks -c ${{ parameters.configuration }} -v:n -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/extract-workloads.binlog
 
 - template: install-apkdiff.yaml
+
+- ${{ if eq(parameters.installTestSlicer, true) }}:
+  - template: install-dotnet-test-slicer.yaml

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -0,0 +1,152 @@
+# Runs MSBuild tests against a device running on macOS
+
+parameters:
+  job_name: 'mac_dotnetdevice_tests'
+  agent_count: 6
+  nunit_categories: $(DotNetNUnitCategories)
+  target_framework: $(DotNetStableTargetFramework)
+  provisionatorChannel: latest
+  stageCondition: succeeded()
+  jobCondition: succeeded()
+
+stages:
+- stage: msbuilddevice_tests
+  displayName: MSBuild Emulator Tests
+  dependsOn: mac_build
+  condition: ${{ parameters.stageCondition }}
+  jobs:
+  - job: ${{ parameters.job_name }}
+    strategy:
+      parallel: ${{ parameters.agent_count }}
+    displayName: "macOS > Tests > MSBuild+Emulator"
+    condition: ${{ parameters.jobCondition }}
+    pool:
+      vmImage: $(HostedMacImage)
+    timeoutInMinutes: 90
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    steps:
+    - template: setup-test-environment.yaml
+      parameters:
+        provisionClassic: false
+        provisionatorChannel: ${{ parameters.provisionatorChannel }}
+        installTestSlicer: true
+
+    - template: run-xaprepare.yaml
+      parameters:
+        displayName: install emulator
+        arguments: --s=EmulatorTestDependencies --android-sdk-platforms="19,21,26,32,33"
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifactName: $(TestAssembliesArtifactName)
+        downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
+
+    - pwsh: |
+        dotnet-test-slicer `
+          --test-assembly="$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll" `
+          --test-filter="cat != TimeZoneInfo & cat != Localization ${{ parameters.nunit_categories }}" `
+          --slice-number=$(System.JobPositionInPhase) `
+          --total-slices=$(System.TotalJobsInPhase) `
+          --outfile="$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.runsettings"
+      displayName: Slice unit tests
+
+    - task: MSBuild@1
+      displayName: start emulator
+      inputs:
+        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
+        configuration: $(XA.Build.Configuration)
+        msbuildArguments: /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
+
+    - template: run-nunit-tests.yaml
+      parameters:
+        useDotNet: true
+        testRunTitle: MSBuildDeviceIntegration On Device - macOS-$(System.JobPositionInPhase)
+        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll
+        dotNetTestExtraArgs: --settings "$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.runsettings"
+        testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-$(System.JobPositionInPhase)-$(XA.Build.Configuration).xml
+
+    - task: MSBuild@1
+      displayName: shut down emulator
+      inputs:
+        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
+        configuration: $(XA.Build.Configuration)
+        msbuildArguments: >-
+          /t:AcquireAndroidTarget,ReleaseAndroidTarget
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
+      condition: always()
+
+    - template: upload-results.yaml
+      parameters:
+        artifactName: Test Results - MSBuild With Emulator - macOS-$(System.JobPositionInPhase)
+
+    - template: fail-on-issue.yaml
+
+  - job: wear_tests
+    displayName: macOS > Tests > WearOS 
+    timeoutInMinutes: 180
+    cancelTimeoutInMinutes: 2
+    strategy:
+      matrix:
+        Android30-x86:
+          avdApiLevel: 30
+          avdAbi: x86
+          avdType: android-wear
+          deviceName: wear_square
+    pool:
+      vmImage: $(HostedMacImage)
+    workspace:
+      clean: all
+    steps:
+    - template: setup-test-environment.yaml
+      parameters:
+        configuration: $(XA.Build.Configuration)
+
+    - template: run-xaprepare.yaml
+      parameters:
+        displayName: install required brew tools and prepare java.interop
+        arguments: --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes
+
+    - template: run-xaprepare.yaml
+      parameters:
+        displayName: install emulator
+        arguments: --s=EmulatorTestDependencies
+
+    - script: echo "##vso[task.setvariable variable=Java8SdkDirectory]$JAVA_HOME_8_X64"
+      displayName: set Java8SdkDirectory
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifactName: $(TestAssembliesArtifactName)
+        downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
+
+    - task: MSBuild@1
+      displayName: install and launch emulator
+      inputs:
+        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
+        configuration: $(XA.Build.Configuration)
+        msbuildArguments: /t:InstallAvdImage;AcquireAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-$(avdApiLevel).binlog
+
+    - template: run-nunit-tests.yaml
+      parameters:
+        testRunTitle: WearOS On Device - macOS
+        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
+        dotNetTestExtraArgs: --filter "TestCategory = WearOS"
+        testResultsFile: TestResult-WearOS--$(XA.Build.Configuration).xml
+
+    - task: MSBuild@1
+      displayName: shut down emulator
+      inputs:
+        solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
+        configuration: $(XA.Build.Configuration)
+        msbuildArguments: /t:AcquireAndroidTarget,ReleaseAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
+      condition: always()
+
+    - template: upload-results.yaml
+      parameters:
+        configuration: $(XA.Build.Configuration)
+        artifactName: Test Results - Emulator $(avdApiLevel)-$(avdAbi)-$(avdType) - macOS
+
+    - template: fail-on-issue.yaml
+

tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs

@@ -6,7 +6,7 @@
 
 namespace Xamarin.Android.Build.Tests
 {
-	[Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT"), Category ("Node-3")]
+	[Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT")]
 	public class AotProfileTests : DeviceTest
 	{
 

tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs

@@ -11,7 +11,7 @@ namespace Xamarin.Android.Build.Tests
 {
 	[TestFixture]
 	[TestFixtureSource(nameof(FixtureArgs))]
-	[Category ("Node-1"), Category ("XamarinBuildDownload")]
+	[Category ("XamarinBuildDownload")]
 	public class BundleToolTests : DeviceTest
 	{
 		static readonly object[] FixtureArgs = {

tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs

@@ -46,7 +46,6 @@ int FindTextInFile (string file, string text)
 		}
 
 		[Test]
-		[Category ("Node-3")]
 		public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isRelease, [Values (false, true)] bool extractNativeLibs, [Values (false, true)] bool useEmbeddedDex)
 		{
 			AssertHasDevices ();
@@ -84,7 +83,6 @@ public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isReleas
 		}
 
 		[Test]
-		[Category ("Node-3")]
 		public void ClassLibraryMainLauncherRuns ([Values (true, false)] bool preloadAssemblies)
 		{
 			AssertHasDevices ();
@@ -169,7 +167,7 @@ public void ClassLibraryMainLauncherRuns ([Values (true, false)] bool preloadAss
 		};
 #pragma warning restore 414
 
-		[Test, Category ("Debugger"), Category ("Node-4")]
+		[Test, Category ("Debugger")]
 		[TestCaseSource (nameof (DebuggerCustomAppTestCases))]
 		[Retry(5)]
 		public void CustomApplicationRunsWithDebuggerAndBreaks (bool embedAssemblies, string fastDevType, bool activityStarts)
@@ -347,7 +345,7 @@ public override void OnCreate ()
 		};
 #pragma warning restore 414
 
-		[Test, Category ("SmokeTests"), Category ("Debugger"), Category ("Node-4")]
+		[Test, Category ("SmokeTests"), Category ("Debugger")]
 		[TestCaseSource (nameof(DebuggerTestCases))]
 		[Retry (5)]
 		public void ApplicationRunsWithDebuggerAndBreaks (bool embedAssemblies, string fastDevType, bool allowDeltaInstall, string username, string debugType)

tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs

@@ -6,7 +6,7 @@
 namespace Xamarin.Android.Build.Tests
 {
 	[TestFixture]
-	[Category ("DotNetIgnore"), Category ("Node-1")] // .csproj files are legacy projects that won't build under dotnet
+	[Category ("DotNetIgnore")] // .csproj files are legacy projects that won't build under dotnet
 	public class DeleteBinObjTest : DeviceTest
 	{
 		const string BaseUrl = "https://github.com/dellis1972/xamarin-android-unittest-files/blob/main/";

tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs

@@ -20,7 +20,7 @@
 namespace Xamarin.Android.Build.Tests
 {
 	[TestFixture]
-	[Category ("UsesDevice"), Category ("Node-2")]
+	[Category ("UsesDevice")]
 	[NonParallelizable]
 	public class DeploymentTest : DeviceTest {
 

tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs

@@ -8,7 +8,7 @@
 namespace Xamarin.Android.Build.Tests
 {
 	[TestFixture]
-	[Category ("UsesDevice"), Category ("Node-2")]
+	[Category ("UsesDevice")]
 	public class InstallAndRunTests : DeviceTest
 	{
 		static ProjectBuilder builder;

tests/MSBuildDeviceIntegration/Tests/InstallTests.cs

@@ -12,7 +12,7 @@
 namespace Xamarin.Android.Build.Tests
 {
 	[TestFixture]
-	[Category ("Commercial"), Category ("UsesDevice"), Category ("Node-1")]
+	[Category ("Commercial"), Category ("UsesDevice")]
 	public class InstallTests : DeviceTest
 	{
 		string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCommand = true)