Use a faster agent

scripts/azure-pipelines-complete-internal.yml

@@ -39,6 +39,19 @@ parameters:
         name: Azure Pipelines
         vmImage: ubuntu-20.04
         os: linux
+  - name: buildAgentAndroidTests
+    displayName: 'The build agent configuration for building the Android tests:'
+    type: object
+    default:
+      pool:
+        name: VSEng-VSMac-Xamarin-Shared
+        vmImage: VSEng-VSMac-Xamarin-Shared
+        os: macos
+        demands:
+          - macOS.Name -equals Ventura
+          - macOS.Architecture -equals x64
+      parameters:
+        installXcode: false
   - name: enableSigning
     displayName: 'Enable package signing (Test signing)'
     type: boolean
@@ -110,3 +123,4 @@ extends:
           buildAgentMacNative: ${{ parameters.buildAgentMac }}
           buildAgentLinux: ${{ parameters.buildAgentLinux }}
           buildAgentLinuxNative: ${{ parameters.buildAgentLinux }}
+          buildAgentAndroidTests: ${{ parameters.buildAgentAndroidTests }}

scripts/azure-pipelines-complete.yml

@@ -65,3 +65,4 @@ extends:
           buildAgentMacNative: ${{ parameters.buildAgentMac }}
           buildAgentLinux: ${{ parameters.buildAgentLinux }}
           buildAgentLinuxNative: ${{ parameters.buildAgentLinux }}
+          buildAgentAndroidTests: ${{ parameters.buildAgentMac }}

scripts/azure-pipelines-tests.yml

@@ -35,6 +35,19 @@ parameters:
         name: Azure Pipelines
         vmImage: ubuntu-20.04
         os: linux
+  - name: buildAgentAndroidTests
+    displayName: 'The build agent configuration for building the Android tests:'
+    type: object
+    default:
+      pool:
+        name: VSEng-VSMac-Xamarin-Shared
+        vmImage: VSEng-VSMac-Xamarin-Shared
+        os: macos
+        demands:
+          - macOS.Name -equals Ventura
+          - macOS.Architecture -equals x64
+      parameters:
+        installXcode: false
 
 variables:
   - template: /scripts/azure-pipelines-variables.yml@self
@@ -65,4 +78,5 @@ extends:
           buildAgentMac: ${{ parameters.buildAgentMac }}
           buildAgentMacNative: ${{ parameters.buildAgentMac }}
           buildAgentLinux: ${{ parameters.buildAgentLinux }}
-          buildAgentLinuxNative: ${{ parameters.buildAgentLinux }}
+          buildAgentLinuxNative: ${{ parameters.buildAgentLinux }}
+          buildAgentAndroidTests: ${{ parameters.buildAgentAndroidTests }}

scripts/azure-pipelines.yml

@@ -113,3 +113,4 @@ extends:
           buildAgentMacNative: ${{ parameters.buildAgentMac }}
           buildAgentLinux: ${{ parameters.buildAgentLinux }}
           buildAgentLinuxNative: ${{ parameters.buildAgentLinux }}
+          buildAgentAndroidTests: ${{ parameters.buildAgentMac }}

scripts/azure-templates-bootstrapper.yml

@@ -30,6 +30,7 @@ parameters:
   installLlvm: true                             # whether or not to install the LLVM compiler
   installEmsdk: false                           # whether or not to install the Emscripten SDK
   installNinja: false                           # whether or not to install the ninja build system
+  installXcode: true                            # whether or not to install Xcode
   publishArtifacts: []                          # the additional artifacts to publish
   tools: []                                     # any additional .net global tools
   skipInstall: false                            # whether or not to install any tools
@@ -105,7 +106,7 @@ jobs:
         - ${{ parameters.provisioningSteps }}
 
       # install any packages on linux
-      - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}:
+      - ${{ if and(eq(parameters.docker, ''), eq(parameters.buildAgent.pool.os, 'linux'), ne(parameters.skipInstall, 'true')) }}:
         - bash: |
             sudo apt update
             sudo apt install -y ${{ parameters.packages }}
@@ -161,7 +162,7 @@ jobs:
             retryCountOnTaskFailure: 3
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
         # install the bits needed for Android on macOS and Windows
-        - ${{ if and(eq(parameters.installAndroidSdk, 'true'), not(endsWith(parameters.name, '_linux'))) }}:
+        - ${{ if and(eq(parameters.installAndroidSdk, 'true'), ne(parameters.buildAgent.pool.os, 'linux')) }}:
           # install the correct version of the JDK
           - pwsh: .\scripts\install-openjdk.ps1
             displayName: Install OpenJDK
@@ -187,7 +188,7 @@ jobs:
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
         # install the bits needed for .NET
         - ${{ if eq(parameters.installDotNet, 'true') }}:
-          - ${{ if endsWith(parameters.name, '_macos') }}:
+          - ${{ if eq(parameters.buildAgent.pool.os, 'macos') }}:
             - pwsh: Remove-Item "$env:AGENT_TOOLSDIRECTORY/dotnet" -Recurse -Force -ErrorAction SilentlyContinue
               displayName: Cleanup existing versions of .NET
               condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
@@ -202,7 +203,7 @@ jobs:
             displayName: Install the preview version of .NET
             retryCountOnTaskFailure: 3
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''), ne(variables.DOTNET_VERSION_PREVIEW, ''))
-          - ${{ if endsWith(parameters.name, '_linux') }}:
+          - ${{ if eq(parameters.buildAgent.pool.os, 'linux') }}:
             - pwsh: .\scripts\patch-dotnet.ps1 -InstallDir "$env:AGENT_TOOLSDIRECTORY/dotnet"
               displayName: Fix the Microsoft.WinFX.* file
               condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
@@ -227,7 +228,7 @@ jobs:
             displayName: Install the .NET workloads
 
       # install the mac tools
-      - ${{ if and(endsWith(parameters.name, '_macos'), ne(parameters.skipInstall, 'true')) }}:
+      - ${{ if and(eq(parameters.installXcode, 'true'), ne(parameters.buildAgent.parameters.installXcode, 'false'), eq(parameters.buildAgent.pool.os, 'macos'), ne(parameters.skipInstall, 'true')) }}:
         - ${{ if not(startsWith(parameters.name, 'native_')) }}:
           - bash: sudo ./scripts/select-xcode.sh $(XCODE_VERSION)
             displayName: Switch to the latest Xcode
@@ -238,14 +239,14 @@ jobs:
             condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
 
       # install the linux tools
-      - ${{ if and(eq(parameters.installEmsdk, 'true'), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}:
+      - ${{ if and(eq(parameters.installEmsdk, 'true'), eq(parameters.buildAgent.pool.os, 'linux'), ne(parameters.skipInstall, 'true')) }}:
         - bash: ./scripts/install-emsdk.sh $(EMSCRIPTEN_VERSION)
           displayName: Install the Emscripten SDK
           retryCountOnTaskFailure: 3
           condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
 
       # install the Windows tools
-      - ${{ if and(endsWith(parameters.name, '_windows'), ne(parameters.skipInstall, 'true')) }}:
+      - ${{ if and(eq(parameters.buildAgent.pool.os, 'windows'), ne(parameters.skipInstall, 'true')) }}:
         # select the correct/latest version of Visual Studio
         - pwsh: .\scripts\select-vs.ps1
           displayName: Select Visual Studio
@@ -299,7 +300,7 @@ jobs:
       # actual build
       - ${{ if ne(parameters.skipSteps, 'true') }}:
         - ${{ if eq(parameters.docker, '') }}:
-          - ${{ if endsWith(parameters.name, '_windows') }}:
+          - ${{ if eq(parameters.buildAgent.pool.os, 'windows') }}:
             - pwsh: |
                 Get-Content $PSCommandPath
                 dotnet tool restore
@@ -314,7 +315,7 @@ jobs:
               displayName: Run the bootstrapper for ${{ parameters.target }}
               retryCountOnTaskFailure: ${{ parameters.retryCount }}
               condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''))
-          - ${{ if not(endsWith(parameters.name, '_windows')) }}:
+          - ${{ if ne(parameters.buildAgent.pool.os, 'windows') }}:
             - bash: |
                 cat ${BASH_SOURCE[0]}
                 dotnet tool restore

scripts/azure-templates-stages.yml

@@ -821,7 +821,7 @@ stages:
             name: tests_android_macos
             displayName: Android (macOS)
             buildPipelineType: ${{ parameters.buildPipelineType }}
-            buildAgent: ${{ parameters.buildAgentMac }}
+            buildAgent: ${{ parameters.buildAgentAndroidTests }}
             target: tests-android
             additionalArgs: --device=android-emulator-64 --deviceVersion=$(ANDROID_TEST_DEVICE_VERSION) --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE)
             shouldPublish: false

scripts/cake/shared.cake

@@ -122,3 +122,14 @@ void CleanDir(DirectoryPath dir)
 
     EnsureDirectoryExists(dir);
 }
+
+void TakeSnapshot(DirectoryPath output, string name)
+{
+    if (IsRunningOnWindows())
+        return;
+
+    var fname = $"screenshot-{DateTime.Now:yyyyMMdd_hhmmss}-{name}.jpg";
+    var dest = output.CombineWithFilePath(fname);
+
+    RunProcess("screencapture", dest.FullPath);
+}

scripts/cake/xharness-android.cake

@@ -18,6 +18,15 @@ var DEVICE_NAME = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "
 var DEVICE_ID = "";
 var DEVICE_ARCH = "";
 
+if (string.IsNullOrEmpty(TEST_APP)) {
+    throw new Exception("A path to a test app is required.");
+}
+if (string.IsNullOrEmpty(TEST_RESULTS)) {
+    TEST_RESULTS = TEST_APP + "-results";
+}
+Information("Test Results Directory: {0}", TEST_RESULTS);
+CleanDir(TEST_RESULTS);
+
 // set up env
 var ANDROID_SDK_ROOT = Argument("android", EnvironmentVariable("ANDROID_SDK_ROOT") ?? EnvironmentVariable("ANDROID_SDK_HOME"));
 if (string.IsNullOrEmpty(ANDROID_SDK_ROOT)) {
@@ -46,7 +55,15 @@ var adbSettings = new AdbToolSettings {
 var emuSettings = new AndroidEmulatorToolSettings {
     SdkRoot = ANDROID_SDK_ROOT,
     ToolPath = $"{ANDROID_SDK_ROOT}/emulator/emulator{exe}",
-    ArgumentCustomization = args => args.Append("-no-window")
+    Verbose = true,
+    ArgumentCustomization = args => args.Append(
+        "-no-boot-anim " +
+        "-no-snapshot " +
+        "-gpu host " +
+        "-no-audio " +
+        "-camera-back none " +
+        "-camera-front none " +
+        "-qemu -m 2048")
 };
 
 AndroidEmulatorProcess emulatorProcess = null;
@@ -115,20 +132,26 @@ Setup(context =>
     Information("Starting Emulator: {0}...", ANDROID_AVD);
     emulatorProcess = AndroidEmulatorStart(ANDROID_AVD, emuSettings);
 
-    // wait for it to finish booting (10 mins)
+    // wait for it to finish booting (4 mins)
     var waited = 0;
-    var total = 60 * 10;
+    var interval = 10;
+    var totalMins = 4;
+    var total = 60 * totalMins / interval;
     while (AdbShell("getprop sys.boot_completed", adbSettings).FirstOrDefault() != "1") {
-        System.Threading.Thread.Sleep(1000);
-        Information("Wating {0}/{1} seconds for the emulator to boot up.", waited, total);
+        TakeSnapshot(TEST_RESULTS, $"boot-{waited:000}");
+        System.Threading.Thread.Sleep(interval * 1000);
+        Information("Wating {0}/{1} seconds for the emulator to boot up.", waited * interval, total);
         if (waited++ > total)
             break;
     }
+    TakeSnapshot(TEST_RESULTS, "boot-complete");
     Information("Waited {0} seconds for the emulator to boot up.", waited);
 });
 
 Teardown(context =>
 {
+    TakeSnapshot(TEST_RESULTS, "teardown");
+
     // no virtual device was used
     if (emulatorProcess == null)
         return;
@@ -150,9 +173,6 @@ Teardown(context =>
 Task("Default")
     .Does(() =>
 {
-    if (string.IsNullOrEmpty(TEST_APP)) {
-        throw new Exception("A path to a test app is required.");
-    }
     if (string.IsNullOrEmpty(TEST_APP_PACKAGE_NAME)) {
         var appFile = (FilePath)TEST_APP;
         appFile = appFile.GetFilenameWithoutExtension();
@@ -161,24 +181,35 @@ Task("Default")
     if (string.IsNullOrEmpty(TEST_APP_INSTRUMENTATION)) {
         TEST_APP_INSTRUMENTATION = TEST_APP_PACKAGE_NAME + ".TestInstrumentation";
     }
-    if (string.IsNullOrEmpty(TEST_RESULTS)) {
-        TEST_RESULTS = TEST_APP + "-results";
-    }
 
     Information("Test App: {0}", TEST_APP);
     Information("Test App Package Name: {0}", TEST_APP_PACKAGE_NAME);
     Information("Test App Instrumentation: {0}", TEST_APP_INSTRUMENTATION);
     Information("Test Results Directory: {0}", TEST_RESULTS);
 
-    CleanDirectories(TEST_RESULTS);
+    TakeSnapshot(TEST_RESULTS, "starting-tests");
+
+    var complete = false;
+    System.Threading.Tasks.Task.Run(() => {
+        while (!complete) {
+            TakeSnapshot(TEST_RESULTS, "running-tests");
+            System.Threading.Thread.Sleep(5000);
+        }
+    });
 
     DotNetTool("xharness android test " +
         $"--app=\"{TEST_APP}\" " +
         $"--package-name=\"{TEST_APP_PACKAGE_NAME}\" " +
         $"--instrumentation=\"{TEST_APP_INSTRUMENTATION}\" " +
         $"--output-directory=\"{TEST_RESULTS}\" " +
+        $"--timeout=00:15:00 " +
+        $"--launch-timeout=00:05:00 " +
         $"--verbosity=\"Debug\" ");
 
+    complete = true;
+
+    TakeSnapshot(TEST_RESULTS, "finished-tests");
+
     var failed = XmlPeek($"{TEST_RESULTS}/TestResults.xml", "/assemblies/assembly[@failed > 0 or @errors > 0]/@failed");
     if (!string.IsNullOrEmpty(failed)) {
         throw new Exception($"At least {failed} test(s) failed.");

scripts/install-openjdk.ps1

@@ -6,7 +6,7 @@ Param(
 
 $ErrorActionPreference = 'Stop'
 
-if (Test-Path (Join-Path "$env:JAVA_HOME_17_X64" "bin")) {
+if ("$env:JAVA_HOME_17_X64" -and (Test-Path (Join-Path "$env:JAVA_HOME_17_X64" "bin"))) {
     Write-Host "Java is already installed to '$env:JAVA_HOME_17_X64'..."
     $java_home = $env:JAVA_HOME_17_X64
 } else {