Skip to content

Commit

Permalink
[ci] Reduce overhead for MSBuildIntegration unit test jobs. (#7832)
Browse files Browse the repository at this point in the history
There are 2 opportunities for speeding up the `MSBuildIntegration` unit test stage: removing `SmokeTests` that we also run in a different stage, and opting out of installing dependencies that these jobs do not need.

* Don't run `SmokeTests` in this stage, as they are run in the dedicated `Smoke Tests` stage.
* Refactor `start emulator` and `stop emulator` into reusable templates, and update them to use `dotnet build` instead of `mono`.
* Opt out of installing unneeded dependencies:
  * nunit-console
  * .NET 6
  * apkdiff
  * Unneeded Android SDK Platforms: 19,21,26,32
  * Mono: theoretically this stage isn't using Mono anymore. Regardless, [Mono is now included in the default Mac VM image](actions/runner-images#6799) (it's a newer version (`6.12.0.188`) than we are installing (`6.12.0.145`)), so we don't need this unless we decide in the future we need a newer version than the default.
    * By updating our required Mono version to `6.12.0.188`, `xaprepare` will not need to downgrade it anymore on the current Mac images, saving time for *all* jobs that install Mono.
  • Loading branch information
jpobst committed Mar 2, 2023
1 parent 1ca7ac7 commit 6fec194
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 56 deletions.
4 changes: 2 additions & 2 deletions Configuration.props
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@
<AndroidSupportedTargetJitAbis Condition=" '$(AndroidSupportedTargetJitAbis)' == '' ">armeabi-v7a:arm64-v8a:x86:x86_64</AndroidSupportedTargetJitAbis>
<JavaInteropSourceDirectory Condition=" '$(JavaInteropSourceDirectory)' == '' ">$(MSBuildThisFileDirectory)external\Java.Interop</JavaInteropSourceDirectory>
<MonoSourceDirectory>$(MSBuildThisFileDirectory)external\mono</MonoSourceDirectory>
<MonoDarwinPackageUrl>https://xamjenkinsartifact.azureedge.net/build-package-osx-mono/2020-02/151/c633fe923832f0c3db3c4e6aa98e5592bf5a06e7/MonoFramework-MDK-6.12.0.145.macos10.xamarin.universal.pkg</MonoDarwinPackageUrl>
<MonoRequiredMinimumVersion Condition=" '$(MonoRequiredMinimumVersion)' == '' ">6.12.0.145</MonoRequiredMinimumVersion>
<MonoDarwinPackageUrl>https://download.mono-project.com/archive/6.12.0/macos-10-universal/MonoFramework-MDK-6.12.0.188.macos10.xamarin.universal.pkg</MonoDarwinPackageUrl>
<MonoRequiredMinimumVersion Condition=" '$(MonoRequiredMinimumVersion)' == '' ">6.12.0.188</MonoRequiredMinimumVersion>
<MonoRequiredMaximumVersion Condition=" '$(MonoRequiredMaximumVersion)' == '' ">$(MonoRequiredMinimumVersion)</MonoRequiredMaximumVersion>
<IgnoreMaxMonoVersion Condition=" '$(IgnoreMaxMonoVersion)' == '' And '$(RunningOnCI)' == 'true' ">False</IgnoreMaxMonoVersion>
<IgnoreMaxMonoVersion Condition=" '$(IgnoreMaxMonoVersion)' == '' ">True</IgnoreMaxMonoVersion>
Expand Down
51 changes: 30 additions & 21 deletions build-tools/automation/yaml-templates/setup-test-environment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ parameters:
jdkTestFolder: $(JAVA_HOME_11_X64)
remove_dotnet: false
installTestSlicer: false
installApkDiff: true
installLegacyDotNet: true
restoreNUnitConsole: true
updateMono: true
androidSdkPlatforms: 19,21,26,32,33

steps:
- checkout: self
Expand Down Expand Up @@ -40,11 +45,12 @@ steps:
condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT'))

# Install .NET 6 for legacy tests
- template: use-dot-net.yaml
parameters:
version: 6.0
quality: GA
remove_dotnet: ${{ parameters.remove_dotnet }}
- ${{ if eq(parameters.installLegacyDotNet, true) }}:
- template: use-dot-net.yaml
parameters:
version: 6.0
quality: GA
remove_dotnet: ${{ parameters.remove_dotnet }}

# Install latest .NET
- template: use-dot-net.yaml
Expand All @@ -56,26 +62,28 @@ steps:
custom: build-server
arguments: shutdown

- template: run-xaprepare.yaml
parameters:
displayName: run xaprepare-UpdateMono
arguments: --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes
condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))
xaSourcePath: ${{ parameters.xaSourcePath }}
- ${{ if eq(parameters.updateMono, true) }}:
- template: run-xaprepare.yaml
parameters:
displayName: run xaprepare-UpdateMono
arguments: --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes
condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))
xaSourcePath: ${{ parameters.xaSourcePath }}

- template: run-xaprepare.yaml
parameters:
arguments: --s=AndroidTestDependencies --android-sdk-platforms="19,21,26,32,33"
arguments: --s=AndroidTestDependencies --android-sdk-platforms="${{ parameters.androidSdkPlatforms }}"
xaSourcePath: ${{ parameters.xaSourcePath }}

- task: DotNetCoreCLI@2
displayName: restore NUnit.Console
inputs:
command: restore
projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj
restoreArguments: -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/restore-Xamarin.ProjectTools.binlog
nugetConfigPath: ${{ parameters.xaSourcePath }}/NuGet.config
feedsToUse: config
- ${{ if eq(parameters.restoreNUnitConsole, true) }}:
- task: DotNetCoreCLI@2
displayName: restore NUnit.Console
inputs:
command: restore
projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj
restoreArguments: -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/restore-Xamarin.ProjectTools.binlog
nugetConfigPath: ${{ parameters.xaSourcePath }}/NuGet.config
feedsToUse: config

- task: DotNetCoreCLI@2
displayName: build Xamarin.Android.Tools.BootstrapTasks.csproj
Expand All @@ -101,7 +109,8 @@ steps:
projects: ${{ parameters.xaSourcePath }}/build-tools/create-packs/Microsoft.Android.Sdk.proj
arguments: -t:ExtractWorkloadPacks -c ${{ parameters.configuration }} -v:n -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/extract-workloads.binlog

- template: install-apkdiff.yaml
- ${{ if eq(parameters.installApkDiff, true) }}:
- template: install-apkdiff.yaml

- ${{ if eq(parameters.installTestSlicer, true) }}:
- template: install-dotnet-test-slicer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,24 @@ stages:
cancelTimeoutInMinutes: 5
workspace:
clean: all
variables:
androidSdkPlatforms: 33
steps:
- template: setup-test-environment.yaml
parameters:
provisionClassic: false
provisionatorChannel: ${{ parameters.provisionatorChannel }}
installTestSlicer: true
installApkDiff: false
installLegacyDotNet: false
restoreNUnitConsole: false
updateMono: false
androidSdkPlatforms: $(androidSdkPlatforms)

- template: run-xaprepare.yaml
parameters:
displayName: install emulator
arguments: --s=EmulatorTestDependencies --android-sdk-platforms="19,21,26,32,33"
arguments: --s=EmulatorTestDependencies --android-sdk-platforms="$(androidSdkPlatforms)"

- task: DownloadPipelineArtifact@2
inputs:
Expand All @@ -46,18 +53,13 @@ stages:
- pwsh: |
dotnet-test-slicer `
--test-assembly="$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll" `
--test-filter="cat != TimeZoneInfo & cat != Localization ${{ parameters.nunit_categories }}" `
--test-filter="cat != TimeZoneInfo & cat != Localization & cat != SmokeTests ${{ parameters.nunit_categories }}" `
--slice-number=$(System.JobPositionInPhase) `
--total-slices=$(System.TotalJobsInPhase) `
--outfile="$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.runsettings"
displayName: Slice unit tests
- task: MSBuild@1
displayName: start emulator
inputs:
solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
- template: start-stop-emulator.yaml

- template: run-nunit-tests.yaml
parameters:
Expand All @@ -67,15 +69,9 @@ stages:
dotNetTestExtraArgs: --settings "$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.runsettings"
testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-$(System.JobPositionInPhase)-$(XA.Build.Configuration).xml

- task: MSBuild@1
displayName: shut down emulator
inputs:
solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: >-
/t:AcquireAndroidTarget,ReleaseAndroidTarget
/bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
condition: always()
- template: start-stop-emulator.yaml
parameters:
command: stop

- template: upload-results.yaml
parameters:
Expand All @@ -94,14 +90,21 @@ stages:
avdAbi: x86
avdType: android-wear
deviceName: wear_square
androidSdkPlatforms: 33
pool:
vmImage: $(HostedMacImage)
workspace:
clean: all
steps:
- template: setup-test-environment.yaml
parameters:
provisionClassic: false
provisionatorChannel: ${{ parameters.provisionatorChannel }}
configuration: $(XA.Build.Configuration)
installApkDiff: false
installLegacyDotNet: false
restoreNUnitConsole: false
androidSdkPlatforms: $(androidSdkPlatforms)

- template: run-xaprepare.yaml
parameters:
Expand All @@ -111,7 +114,7 @@ stages:
- template: run-xaprepare.yaml
parameters:
displayName: install emulator
arguments: --s=EmulatorTestDependencies
arguments: --s=EmulatorTestDependencies --android-sdk-platforms="$(androidSdkPlatforms)"

- script: echo "##vso[task.setvariable variable=Java8SdkDirectory]$JAVA_HOME_8_X64"
displayName: set Java8SdkDirectory
Expand All @@ -121,12 +124,13 @@ stages:
artifactName: $(TestAssembliesArtifactName)
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)

- task: MSBuild@1
displayName: install and launch emulator
inputs:
solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /t:InstallAvdImage;AcquireAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-$(avdApiLevel).binlog
- template: start-stop-emulator.yaml
parameters:
specificImage: true
deviceName: $(deviceName)
avdApiLevel: $(avdApiLevel)
avdAbi: $(avdAbi)
avdType: $(avdType)

- template: run-nunit-tests.yaml
parameters:
Expand All @@ -135,13 +139,14 @@ stages:
dotNetTestExtraArgs: --filter "TestCategory = WearOS"
testResultsFile: TestResult-WearOS--$(XA.Build.Configuration).xml

- task: MSBuild@1
displayName: shut down emulator
inputs:
solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /t:AcquireAndroidTarget,ReleaseAndroidTarget /p:TestDeviceName=$(deviceName) /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
condition: always()
- template: start-stop-emulator.yaml
parameters:
command: stop
specificImage: true
deviceName: $(deviceName)
avdApiLevel: $(avdApiLevel)
avdAbi: $(avdAbi)
avdType: $(avdType)

- template: upload-results.yaml
parameters:
Expand Down
31 changes: 31 additions & 0 deletions build-tools/automation/yaml-templates/start-stop-emulator.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
parameters:
command: start # 'start' or 'stop'
specificImage: false # 'true' to use a specific emulator AVD image
deviceName: # Device name, like 'wear_square', required if 'specificImage' is 'true'
avdApiLevel: # Device API level, like '30', required if 'specificImage' is 'true'
avdAbi: # Device ABI, like 'x86', required if 'specificImage' is 'true'
avdType: # Device AVD, like 'android-wear', required if 'specificImage' is 'true'

steps:
- ${{ if eq(parameters.command, 'start') }}:
- task: DotNetCoreCLI@2
displayName: Start emulator
continueOnError: false
inputs:
projects: src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
${{ if eq(parameters.specificImage, true) }}:
arguments: -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget" -p:TestDeviceName=${{ parameters.deviceName }} -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} -p:TestAvdAbi=${{ parameters.avdAbi }} -p:TestAvdType=${{ parameters.avdType }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/install-emulator-${{ parameters.avdApiLevel }}.binlog
${{ else }}:
arguments: -c $(XA.Build.Configuration) -t:AcquireAndroidTarget -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog

- ${{ if eq(parameters.command, 'stop') }}:
- task: DotNetCoreCLI@2
displayName: Stop emulator
condition: always()
continueOnError: true
inputs:
projects: src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
${{ if eq(parameters.specificImage, true) }}:
arguments: -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" -p:TestDeviceName=${{ parameters.deviceName }} -p:TestAvdApiLevel=${{ parameters.avdApiLevel }} -p:TestAvdAbi=${{ parameters.avdAbi }} -p:TestAvdType=${{ parameters.avdType }} -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
${{ else }}:
arguments: -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget" -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
<TargetFramework>netstandard2.0</TargetFramework>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
</PropertyGroup>
<Import Project="..\..\Configuration.props" />
<Import Project="..\..\build-tools\scripts\TestApks.targets" />
<Import Project="$(MSBuildThisFileDirectory)..\..\..\..\Configuration.props" />
<Import Project="$(MSBuildThisFileDirectory)..\..\..\..\build-tools\scripts\TestApks.targets" />
</Project>

0 comments on commit 6fec194

Please sign in to comment.