From 07d48649cda2e5a841e14ab1328636ceede0175c Mon Sep 17 00:00:00 2001
From: Jonathan Peppers <jonathan.peppers@microsoft.com>
Date: Thu, 23 Jul 2020 11:33:23 -0500
Subject: [PATCH] WIP attempt to fan out to 3 CI machines

---
 build-tools/automation/azure-pipelines.yaml   | 28 +++++++++-
 .../yaml-templates/run-msbuild-mac-tests.yaml | 43 +++++++++------
 .../yaml-templates/run-msbuild-win-tests.yaml | 52 +++++++++++--------
 3 files changed, 83 insertions(+), 40 deletions(-)

diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml
index 4275873d4bb..ec9d6ac7355 100644
--- a/build-tools/automation/azure-pipelines.yaml
+++ b/build-tools/automation/azure-pipelines.yaml
@@ -827,9 +827,21 @@ stages:
     parameters:
       node_id: 3
 
+  # Xamarin.Android (Test One .NET - macOS)
   - template: yaml-templates\run-msbuild-mac-tests.yaml
     parameters:
-      node_id: 4
+      node_id: 1
+      use_dotnet: true
+
+  - template: yaml-templates\run-msbuild-mac-tests.yaml
+    parameters:
+      node_id: 2
+      use_dotnet: true
+
+  - template: yaml-templates\run-msbuild-mac-tests.yaml
+    parameters:
+      node_id: 3
+      use_dotnet: true
 
   # Xamarin.Android (Test MSBuild - Windows)
   - template: yaml-templates\run-msbuild-win-tests.yaml
@@ -844,9 +856,21 @@ stages:
     parameters:
       node_id: 3
 
+  # Xamarin.Android (Test One .NET - Windows)
   - template: yaml-templates\run-msbuild-win-tests.yaml
     parameters:
-      node_id: 4
+      node_id: 1
+      use_dotnet: true
+
+  - template: yaml-templates\run-msbuild-win-tests.yaml
+    parameters:
+      node_id: 2
+      use_dotnet: true
+
+  - template: yaml-templates\run-msbuild-win-tests.yaml
+    parameters:
+      node_id: 3
+      use_dotnet: true
 
   # Check - "Xamarin.Android (Test MSBuild With Emulator - macOS)"
   - job: mac_msbuilddevice_tests
diff --git a/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml
index c4b4dab1410..9c1c37d2542 100644
--- a/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml
+++ b/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml
@@ -2,10 +2,28 @@
 
 parameters:
   node_id: 0
+  use_dotnet: false
+
+variables:
+  - TestNode: Node-${{ parameters.node_id }}
+  - ${{ if eq(parameters.use_dotnet, true) }}:
+    TestJobName: mac_dotnet_tests${{ parameters.node_id }}
+    TestDisplayName: One .NET - Mac-${{ parameters.node_id }}
+    TestRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }}
+    TestResultsFile: TestResult-DotNetTests-macOS-$(TestNode)-$(XA.Build.Configuration).xml
+    TestArtifactName: Test Results - One .NET - Mac-${{ parameters.node_id }}
+    NUnitArgs: --where "cat == $(TestNode) && cat != DotNetIgnore && cat != AOT && cat != FSharp && cat != LibraryProjectZip && cat != MkBundle && cat != MonoSymbolicate  && cat != PackagesConfig" --params dotnet=true
+  - ${{ if eq(parameters.use_dotnet, false) }}:
+    TestJobName: mac_msbuild_tests${{ parameters.node_id }}
+    TestDisplayName: MSBuild - Mac-${{ parameters.node_id }}
+    TestRunTitle: Xamarin.Android.Build.Tests - Mac-${{ parameters.node_id }} - One .NET
+    TestResultsFile: TestResult-MSBuildTests-Mac-$(TestNode)-$(XA.Build.Configuration).xml
+    TestArtifactName: Test Results - MSBuild - Mac-${{ parameters.node_id }}
+    NUnitArgs: --where "cat == $(TestNode)"
 
 jobs:
-  - job: mac_msbuild_tests${{ parameters.node_id }}
-    displayName: MSBuild - Mac-${{ parameters.node_id }}
+  - job: $(TestJobName)
+    displayName: $(TestDisplayName)
     pool: $(HostedMac)
     timeoutInMinutes: 180
     cancelTimeoutInMinutes: 5
@@ -26,13 +44,13 @@ jobs:
 
     - template: run-nunit-tests.yaml
       parameters:
-        testRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }}
+        testRunTitle: $(TestRunTitle)
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
-        nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }}"
-        testResultsFile: TestResult-MSBuildTests-macOS-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
+        nunitConsoleExtraArgs: $(NUnitArgs)
+        testResultsFile: $(TestResultsFile)
 
-    # Only run these tests on node 2
-    - ${{ if eq(parameters.node_id, 2) }}:
+    # Only run these tests on node 2 without dotnet
+    - ${{ if and(eq(parameters.node_id, 2), eq(parameters.use_dotnet, false)) }}:
       - template: run-nunit-tests.yaml
         parameters:
           testRunTitle: CodeBehindUnitTests - macOS
@@ -64,17 +82,8 @@ jobs:
           nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
           testResultsFile: TestResult-MSBuildTests-macOS-NoNode-$(XA.Build.Configuration).xml
 
-    # Only run these tests on node 4
-    - ${{ if eq(parameters.node_id, 4) }}:
-      - template: run-nunit-tests.yaml
-        parameters:
-          testRunTitle: Xamarin.Android.Build.Tests - macOS - One .NET
-          testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.Build.Tests.dll
-          nunitConsoleExtraArgs: --where "cat != DotNetIgnore && cat != AOT && cat != FSharp && cat != LibraryProjectZip && cat != MkBundle && cat != MonoSymbolicate && cat != PackagesConfig" --params dotnet=true
-          testResultsFile: TestResult-MSBuildTests-macOS-dotnet-$(XA.Build.Configuration).xml
-
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - MSBuild - Mac-${{ parameters.node_id }}
+        artifactName: $(TestArtifactName)
 
     - template: fail-on-issue.yaml
diff --git a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml
index 5c2dde11b75..c43a51eaa1f 100644
--- a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml
+++ b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml
@@ -2,10 +2,31 @@
 
 parameters:
   node_id: 0
+  use_dotnet: false
+
+variables:
+  - TestNode: Node-${{ parameters.node_id }}
+  # Limit the amount of worker threads used to run these tests in parallel to half of what is currently available (8) on the Windows pool.
+  # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically.
+  - NUnitWorkers: --workers=4
+  - ${{ if eq(parameters.use_dotnet, true) }}:
+    TestJobName: win_dotnet_tests${{ parameters.node_id }}
+    TestDisplayName: One .NET - Windows-${{ parameters.node_id }}
+    TestRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }}
+    TestResultsFile: TestResult-DotNetTests-Windows-$(TestNode)-$(XA.Build.Configuration).xml
+    TestArtifactName: Test Results - One .NET - Windows-${{ parameters.node_id }}
+    NUnitArgs: $(NUnitWorkers) --where "cat == $(TestNode) && cat != DotNetIgnore && cat != AOT && cat != FSharp && cat != LibraryProjectZip && cat != MkBundle && cat != MonoSymbolicate  && cat != PackagesConfig" --params dotnet=true
+  - ${{ if eq(parameters.use_dotnet, false) }}:
+    TestJobName: win_msbuild_tests${{ parameters.node_id }}
+    TestDisplayName: MSBuild - Windows-${{ parameters.node_id }}
+    TestRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }} - One .NET
+    TestResultsFile: TestResult-MSBuildTests-Windows-$(TestNode)-$(XA.Build.Configuration).xml
+    TestArtifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }}
+    NUnitArgs: $(NUnitWorkers) --where "cat == $(TestNode)"
 
 jobs:
-  - job: win_msbuild_tests${{ parameters.node_id }}
-    displayName: MSBuild - Windows-${{ parameters.node_id }}
+  - job: $(TestJobName)
+    displayName: $(TestDisplayName)
     pool: $(VSEngWinVS2019)
     timeoutInMinutes: 180
     cancelTimeoutInMinutes: 5
@@ -33,17 +54,15 @@ jobs:
         artifactName: $(NuGetArtifactName)
         downloadPath: $(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\$(NuGetArtifactName)
 
-    # Limit the amount of worker threads used to run these tests in parallel to half of what is currently available (8) on the Windows pool.
-    # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically.
     - template: run-nunit-tests.yaml
       parameters:
-        testRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }}
+        testRunTitle: $(TestRunTitle)
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
-        nunitConsoleExtraArgs: --workers=4 --where "cat == Node-${{ parameters.node_id }}"
-        testResultsFile: TestResult-MSBuildTests-Windows-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
+        nunitConsoleExtraArgs: $(NUnitArgs)
+        testResultsFile: $(TestResultsFile)
 
-    # Only run these tests on node 2
-    - ${{ if eq(parameters.node_id, 2) }}:
+    # Only run these tests on node 2 without dotnet
+    - ${{ if and(eq(parameters.node_id, 2), eq(parameters.use_dotnet, false)) }}:
       - template: run-nunit-tests.yaml
         parameters:
           testRunTitle: Xamarin.Android.Build.Tests.Commercial - Windows
@@ -54,20 +73,11 @@ jobs:
         parameters:
           testRunTitle: Xamarin.Android.Build.Tests - Windows - No Node
           testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
-          nunitConsoleExtraArgs: --workers=4 --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
-          testResultsFile: TestResult-MSBuildTests-Windows-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
-
-    # Only run these tests on node 4
-    - ${{ if eq(parameters.node_id, 4) }}:
-      - template: run-nunit-tests.yaml
-        parameters:
-          testRunTitle: Xamarin.Android.Build.Tests - Windows - One .NET
-          testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\Xamarin.Android.Build.Tests.dll
-          nunitConsoleExtraArgs: --workers=4 --where "cat != DotNetIgnore && cat != AOT && cat != FSharp && cat != LibraryProjectZip && cat != MkBundle && cat != MonoSymbolicate  && cat != PackagesConfig" --params dotnet=true
-          testResultsFile: TestResult-MSBuildTests-Windows-dotnet-$(XA.Build.Configuration).xml
+          nunitConsoleExtraArgs: $(NUnitWorkers) --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
+          testResultsFile: TestResult-MSBuildTests-Windows-NoNode-$(XA.Build.Configuration).xml
 
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }}
+        artifactName: $(TestArtifactName)
 
     - template: fail-on-issue.yaml