Skip to content

Commit

Permalink
[ci] Move build stages into yaml templates (#7553) (#7720)
Browse files Browse the repository at this point in the history
Context: xamarin/sdk-insertions#42
Context: https://devdiv.visualstudio.com/DevDiv/_build?definitionId=17671&_a=summary

As part of efforts to create an integration and release pipeline for all
of MAUI we will want to "templatize" more of our YAML.  Our three build
stages have been moved into new templates that can be referenced by the
MAUI "mega pipeline".

In an attempt to unify the build for all the SDKs we need to rename all
the resources to use the same names.

Co-authored-by: Manuel de la Pena <mandel@microsoft.com>
  • Loading branch information
pjcollins and mandel-macaque committed Jan 20, 2023
1 parent 402a7f1 commit 4e09bf4
Show file tree
Hide file tree
Showing 8 changed files with 348 additions and 270 deletions.
2 changes: 1 addition & 1 deletion build-tools/automation/azure-pipelines-nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pr:
# External sources, scripts, tests, and yaml template files.
resources:
repositories:
- repository: yaml
- repository: yaml-templates
type: github
name: xamarin/yaml-templates
ref: refs/heads/main
Expand Down
284 changes: 21 additions & 263 deletions build-tools/automation/azure-pipelines.yaml

Large diffs are not rendered by default.

91 changes: 91 additions & 0 deletions build-tools/automation/yaml-templates/build-linux.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
parameters:
buildPool: android-devdiv-ubuntu-vmss
buildResultArtifactName: Build Results - Linux
checkoutCommit: ''
checkoutPath: 's/xamarin-android'
checkoutPersistCredentials: false
dependsOn: ''
dependsOnResult: ''
jobName: linux_build_create_sdk_pack
jobDisplayName: Linux > Build
nugetArtifactName: $(LinuxNuGetArtifactName)
repositoryAlias: self
stageName: linux_build
stageDisplayName: Linux

stages:
- stage: ${{ parameters.stageName }}
displayName: ${{ parameters.stageDisplayName }}
dependsOn: ${{ parameters.dependsOn }}
${{ if and(ne(parameters.dependsOn, ''), ne(parameters.dependsOnResult, '')) }}:
condition: eq(dependencies.${{ parameters.dependsOn }}.result, '${{ parameters.dependsOnResult }}')
jobs:
- job: ${{ parameters.jobName }}
displayName: ${{ parameters.jobDisplayName }}
pool: ${{ parameters.buildPool }}
timeoutInMinutes: 180
workspace:
clean: all
variables:
CXX: g++-10
CC: gcc-10
steps:
- template: sdk-unified/steps/checkout/v1.yml@yaml-templates
parameters:
resource: ${{ parameters.repositoryAlias }}
commit: ${{ parameters.checkoutCommit }}
path: ${{ parameters.checkoutPath }}
persistCredentials: ${{ parameters.checkoutPersistCredentials }}

- checkout: monodroid
clean: true
submodules: recursive
path: s/xamarin-android/external/monodroid
persistCredentials: true

- script: rm -rf external/monodroid/external/xamarin-android
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: delete external xamarin-android submodule

- template: setup-ubuntu.yaml

- task: NuGetAuthenticate@0
displayName: authenticate with azure artifacts
inputs:
forceReinstallCredentialProvider: true

- script: make prepare-external-git-dependencies PREPARE_CI=1 CONFIGURATION=$(XA.Build.Configuration)
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: make prepare-external-git-dependencies

- script: make jenkins PREPARE_CI=1 PREPARE_AUTOPROVISION=1 CONFIGURATION=$(XA.Build.Configuration)
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: make jenkins

- script: make create-nupkgs CONFIGURATION=$(XA.Build.Configuration)
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: make create-nupkgs

- script: >
df -h &&
mkdir -p $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/nuget-linux &&
ln $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/$(NuGetArtifactName)/Microsoft.Android.Sdk.Linux*.nupkg
$(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/nuget-linux &&
ln $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/$(NuGetArtifactName)/SignList.xml
$(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/nuget-linux
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: copy linux sdk
- task: PublishPipelineArtifact@1
displayName: upload linux sdk
inputs:
artifactName: ${{ parameters.nugetArtifactName }}
targetPath: $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/nuget-linux

- template: upload-results.yaml
parameters:
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
artifactName: ${{ parameters.buildResultArtifactName }}
includeBuildResults: true

- template: fail-on-issue.yaml
90 changes: 90 additions & 0 deletions build-tools/automation/yaml-templates/build-macos.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
parameters:
buildPoolName: $(MacBuildPoolName)
buildPoolImage: $(MacBuildPoolImage)
buildResultArtifactName: Build Results - macOS
checkoutCommit: ''
checkoutPath: 's/xamarin-android'
checkoutPersistCredentials: false
dependsOn: ''
dependsOnResult: ''
installerArtifactName: $(InstallerArtifactName)
jobName: mac_build_create_installers
jobDisplayName: macOS > Build
nugetArtifactName: $(NuGetArtifactName)
provisionatorChannel: latest
repositoryAlias: self
signClassicPkgContent: false
stageName: mac_build
stageDisplayName: Mac
testAssembliesArtifactName: $(TestAssembliesArtifactName)
windowsToolchainPdbArtifactName: $(WindowsToolchainPdbArtifactName)

stages:
- stage: ${{ parameters.stageName }}
displayName: ${{ parameters.stageDisplayName }}
dependsOn: ${{ parameters.dependsOn }}
${{ if and(ne(parameters.dependsOn, ''), ne(parameters.dependsOnResult, '')) }}:
condition: eq(dependencies.${{ parameters.dependsOn }}.result, '${{ parameters.dependsOnResult }}')
jobs:
# Check - "Xamarin.Android (macOS > Build)"
- job: ${{ parameters.jobName }}
displayName: ${{ parameters.jobDisplayName }}
pool:
name: ${{ parameters.buildPoolName }}
vmImage: ${{ parameters.buildPoolImage }}
${{ if or(and(ne(variables['Build.DefinitionName'],'Xamarin.Android'), ne(variables['Build.DefinitionName'], 'Xamarin.Android-Private')), eq(variables['Build.Reason'], 'PullRequest')) }}:
demands: macOS.Name -equals Monterey
timeoutInMinutes: 240
cancelTimeoutInMinutes: 5
workspace:
clean: all
steps:
- template: sdk-unified/steps/checkout/v1.yml@yaml-templates
parameters:
resource: ${{ parameters.repositoryAlias }}
commit: ${{ parameters.checkoutCommit }}
path: ${{ parameters.checkoutPath }}
persistCredentials: ${{ parameters.checkoutPersistCredentials }}

- template: install-microbuild-tooling.yaml
parameters:
condition: and(succeeded(), eq(variables['MicroBuildSignType'], 'Real'))

- template: commercial-build.yaml
parameters:
installerArtifactName: ${{ parameters.installerArtifactName }}
nugetArtifactName: ${{ parameters.nugetArtifactName }}
provisionatorChannel: ${{ parameters.provisionatorChannel }}
signClassicPkgContent: ${{ parameters.signClassicPkgContent }}
testAssembliesArtifactName: ${{ parameters.testAssembliesArtifactName }}

- template: remove-microbuild-tooling.yaml
parameters:
condition: and(succeededOrFailed(), eq(variables['MicroBuildSignType'], 'Real'))

- script: >
mkdir -p $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/windows-toolchain-pdb &&
cd $(System.DefaultWorkingDirectory)/xamarin-android/bin/$(XA.Build.Configuration)/lib/packs/Microsoft.Android.Sdk.Darwin/*/tools/binutils/windows-toolchain-pdb &&
zip -r $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/windows-toolchain-pdb/windows-toolchain-pdb.zip .
workingDirectory: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: zip Windows toolchain pdb files
- task: PublishPipelineArtifact@1
displayName: upload Windows toolchain pdb files
inputs:
artifactName: ${{ parameters.windowsToolchainPdbArtifactName }}
targetPath: $(System.DefaultWorkingDirectory)/xamarin-android/bin/Build$(XA.Build.Configuration)/windows-toolchain-pdb

- template: upload-results.yaml
parameters:
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
artifactName: ${{ parameters.buildResultArtifactName }}
includeBuildResults: true

- template: run-xaprepare.yaml
parameters:
arguments: --s=DetermineApplicableTests
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
displayName: determine which test stages to run
name: TestConditions
condition: and(succeeded(), eq(variables['Build.DefinitionName'], 'Xamarin.Android-PR'))
136 changes: 136 additions & 0 deletions build-tools/automation/yaml-templates/build-windows.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
parameters:
buildPool: $(1ESWindowsPool)
buildResultArtifactName: Build Results - Windows
checkoutCommit: ''
checkoutPath: ''
checkoutPersistCredentials: false
dependsOn: ''
dependsOnResult: ''
jobName: win_build_test
jobDisplayName: Windows > Build & Smoke Test
repositoryAlias: self
stageName: win_build_test
stageDisplayName: Windows

# This stage ensures Windows specific build steps continue to work, and runs unit tests.
stages:
- stage: ${{ parameters.stageName }}
displayName: ${{ parameters.stageDisplayName }}
dependsOn: ${{ parameters.dependsOn }}
${{ if and(ne(parameters.dependsOn, ''), ne(parameters.dependsOnResult, '')) }}:
condition: eq(dependencies.${{ parameters.dependsOn }}.result, '${{ parameters.dependsOnResult }}')
jobs:
# Check - "Xamarin.Android (Windows > Build & Smoke Test)"
- job: ${{ parameters.jobName }}
displayName: ${{ parameters.jobDisplayName }}
pool: ${{ parameters.buildPool }}
timeoutInMinutes: 360
steps:
- template: sdk-unified/steps/checkout/v1.yml@yaml-templates
parameters:
resource: ${{ parameters.repositoryAlias }}
commit: ${{ parameters.checkoutCommit }}
path: ${{ parameters.checkoutPath }}
persistCredentials: ${{ parameters.checkoutPersistCredentials }}

- template: kill-processes.yaml

- template: clean.yaml

- script: |
echo ##vso[task.setvariable variable=JI_JAVA_HOME]%JAVA_HOME_11_X64%
displayName: set JI_JAVA_HOME
- template: use-dot-net.yaml
parameters:
remove_dotnet: true

# xabuild still depends on .NET Core 3 or earlier
- template: use-dot-net.yaml
parameters:
version: 3.1.417

# Downgrade the XA .vsix installed into the instance of VS that we are building with so that we don't restore/build against a test version.
# The VS installer will attempt to resume any failed or partial installation before trying to downgrade Xamarin.Android.
# VSIXInstaller.exe will exit non-zero when the downgrade attempt is a no-op, so we will allow this step to fail silently.
- powershell: |
$vsWhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
& "$vsWhere" -all -prerelease -latest | Out-Default
$isLatestVSLaunchable = & "$vsWhere" -all -prerelease -latest -property isLaunchable
if ($isLatestVSLaunchable -eq 0) {
$vsPath = & "$vsWhere" -all -prerelease -latest -property installationPath
Write-Host "Attempting to repair VS instance:" $vsPath
$vsInstaller = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vs_installer.exe"
& "$vsInstaller" resume --installPath $vsPath --quiet --norestart | Out-Default
Write-Host "vs_installer.exe resume attempt complete"
}
$vsixInstaller = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\resources\app\ServiceHub\Services\Microsoft.VisualStudio.Setup.Service\VSIXInstaller.exe"
$ts = Get-Date -Format FileDateTimeUniversal
$log = "xavsixdowngrade-$ts.log"
$process = Start-Process -NoNewWindow -FilePath $vsixInstaller -ArgumentList "/downgrade:Xamarin.Android.Sdk /admin /quiet /logFile:$log" -Wait -PassThru -RedirectStandardError "err.txt"
Get-Content "err.txt" | Write-Host
Get-Content "${env:TEMP}\$log" | Write-Host
Write-Host "VSInstaller.exe exited with code:" $process.ExitCode
Remove-Item "${env:TEMP}\$log"
displayName: downgrade XA to stable
ignoreLASTEXITCODE: true
- task: DotNetCoreCLI@2
displayName: Prepare Solution
inputs:
projects: Xamarin.Android.sln
arguments: '-c $(XA.Build.Configuration) -t:Prepare --no-restore -p:AutoProvision=true -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-build-prepare.binlog'

# Build, pack .nupkgs, and extract workload packs to dotnet preview test directory
- template: run-dotnet-preview.yaml
parameters:
project: Xamarin.Android.sln
arguments: >-
-t:BuildDotNet,PackDotNet -c $(XA.Build.Configuration) -v:n
-bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-build.binlog
displayName: Build Solution
continueOnError: false

- task: MSBuild@1
displayName: msbuild create-vsix
inputs:
solution: build-tools\create-vsix\create-vsix.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /p:CreateVsixContainer=True /p:ZipPackageCompressionLevel=Normal /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-create-vsix.binlog

- task: MSBuild@1
displayName: msbuild xabuild
inputs:
solution: tools\xabuild\xabuild.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /restore /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-xabuild.binlog

- task: CmdLine@1
displayName: xabuild Xamarin.Android-Tests
inputs:
filename: bin\$(XA.Build.Configuration)\bin\xabuild.exe
arguments: Xamarin.Android-Tests.sln /restore /p:Configuration=$(XA.Build.Configuration) /bl:$(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\dotnet-build-tests.binlog

- template: install-apkdiff.yaml

- template: run-nunit-tests.yaml
parameters:
testRunTitle: Smoke MSBuild Tests - Windows Build Tree
testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\net472\Xamarin.Android.Build.Tests.dll
testResultsFile: TestResult-SmokeMSBuildTests-WinBuildTree-$(XA.Build.Configuration).xml
nunitConsoleExtraArgs: --where "cat == SmokeTests"

- template: run-nunit-tests.yaml
parameters:
useDotNet: true
testRunTitle: Smoke MSBuild Tests - Windows Dotnet Build
testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\$(DotNetStableTargetFramework)\Xamarin.Android.Build.Tests.dll
testResultsFile: TestResult-SmokeMSBuildTests-WinDotnetBuild-$(XA.Build.Configuration).xml
dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"

- template: upload-results.yaml
parameters:
artifactName: ${{ parameters.buildResultArtifactName }}
includeBuildResults: true

- template: fail-on-issue.yaml
11 changes: 7 additions & 4 deletions build-tools/automation/yaml-templates/commercial-build.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
parameters:
installerArtifactName: $(InstallerArtifactName)
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
makeMSBuildArgs: ''
nugetArtifactName: $(NuGetArtifactName)
provisionatorChannel: latest
signClassicPkgContent: true
testAssembliesArtifactName: $(TestAssembliesArtifactName)

steps:
- script: echo "##vso[task.setvariable variable=JI_JAVA_HOME]$HOME/android-toolchain/jdk-11"
Expand Down Expand Up @@ -118,7 +121,7 @@ steps:
- task: PublishPipelineArtifact@1
displayName: upload nupkgs
inputs:
artifactName: $(NuGetArtifactName)
artifactName: ${{ parameters.nugetArtifactName }}
targetPath: ${{ parameters.xaSourcePath }}/bin/Build$(XA.Build.Configuration)/$(NuGetArtifactName)

- script: >
Expand All @@ -137,7 +140,7 @@ steps:
- task: PublishPipelineArtifact@1
displayName: upload installers
inputs:
artifactName: $(InstallerArtifactName)
artifactName: ${{ parameters.installerArtifactName }}
targetPath: ${{ parameters.xaSourcePath }}/bin/Build$(XA.Build.Configuration)/$(InstallerArtifactName)

# Upload test assemblies
Expand All @@ -150,11 +153,11 @@ steps:
- task: PublishPipelineArtifact@1
displayName: upload test assemblies
inputs:
artifactName: $(TestAssembliesArtifactName)
artifactName: ${{ parameters.testAssembliesArtifactName }}
targetPath: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)

- task: PublishPipelineArtifact@1
displayName: upload build tools inventory
inputs:
artifactName: BuildToolsInventory
artifactName: AndroidBuildToolsInventory
targetPath: ${{ parameters.xaSourcePath }}/bin/Build$(XA.Build.Configuration)/buildtoolsinventory.csv
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ steps:
condition: ${{ parameters.condition }}

# ESRP signing requires minimum azure client version 2.8.0
- template: azure-tools/az-client-update.yml@yaml
- template: azure-tools/az-client-update.yml@yaml-templates
parameters:
version: '2.8.0'
condition: and(${{ parameters.condition }}, eq(variables['agent.os'], 'Darwin'))
Expand Down
2 changes: 1 addition & 1 deletion build-tools/automation/yaml-templates/update-vs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ steps:
provisioning_extra_args: -vv
condition: ${{ parameters.condition }}

- template: environment/win/vs-msbuild.v1.yml@yaml # Display (in log) VS installation(s) including installation status with associated MSBuild location(s)
- template: environment/win/vs-msbuild.v1.yml@yaml-templates # Display (in log) VS installation(s) including installation status with associated MSBuild location(s)
parameters:
condition: ${{ parameters.condition }}

0 comments on commit 4e09bf4

Please sign in to comment.