Skip to content

Commit 19d4c23

Browse files
almiliMSFTalmiliMSFT
committedAug 2, 2022
Merged PR 672509: Migrate 'Linux PR Validation' pipeline to 1es templates
Migrate 'Linux PR Validation' pipeline to 1es templates
1 parent 867f097 commit 19d4c23

File tree

9 files changed

+151
-132
lines changed

9 files changed

+151
-132
lines changed
 

‎.azdo/linux/job-linux-runtime.yml

-75
This file was deleted.

‎.azdo/linux/job-selfhost.yml

+17-10
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,22 @@
1+
parameters:
2+
- name: BxlCommonArgs
3+
type: string
4+
default: --shared-comp /ado /cacheMiss:"[Bxl.Selfhost.Linux]" /logObservedFileAccesses /cacheConfigFilePath:Out/CacheConfig.json /logoutput:FullOutputOnError /logsToRetain:10 /exp:lazysodeletion- # /p:[Sdk.BuildXL]xunitSemaphoreCount=20
5+
16
jobs:
27
- job: Selfhost
38
displayName: Build and Validate Selfhost
49
pool:
510
name: BuildXL-DevOpsAgents-Linux-PME
611

12+
templateContext:
13+
breakGlass:
14+
justification: 'Because BuildXL downloads NuGet packages from the internet during the build (and there is no way to run "nuget restore" before the build because NuGet packages are specified in a BuildXL-specific way)'
15+
inputs:
16+
- input: checkout
17+
repository: self
18+
fetchDepth: 1
19+
720
timeoutInMinutes: 90
821

922
strategy:
@@ -15,13 +28,7 @@ jobs:
1528
BxlMatrixName: ReleaseLinux
1629
BxlMatrixArgs: /q:ReleaseLinux
1730

18-
variables:
19-
skipComponentGovernanceDetection: true # done elsewhere (in the main rolling pipeline)
20-
BxlCommonArgs: --shared-comp /ado /cacheMiss:"[Bxl.Selfhost.Linux]" /logObservedFileAccesses /cacheConfigFilePath:Out/CacheConfig.json /logoutput:FullOutputOnError /logsToRetain:10 /exp:lazysodeletion- # /p:[Sdk.BuildXL]xunitSemaphoreCount=20
21-
2231
steps:
23-
- checkout: self
24-
fetchDepth: 1
2532

2633
# Azure Key Vault
2734
# Download Azure Key Vault secrets
@@ -37,7 +44,7 @@ jobs:
3744
inputs:
3845
version: 6.x
3946

40-
- template: step-nuget-config.yml
47+
- template: step-install-mono.yml
4148

4249
- bash: |
4350
set -euo pipefail
@@ -82,7 +89,7 @@ jobs:
8289
8390
- bash: |
8491
set -eu
85-
bash bxl.sh $(BxlCommonArgs) /logsDirectory:"Out/Logs/Build" --minimal --internal --deploy-dev-release
92+
bash bxl.sh ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/Build" --minimal --internal --deploy-dev-release
8693
displayName: Build
8794
workingDirectory: /home/subst
8895
env:
@@ -91,7 +98,7 @@ jobs:
9198
VSTSPERSONALACCESSTOKEN: $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
9299
93100
- bash: |
94-
echo "== Deleting Out/frontend/Nuget/tmp, Out/bin, Out/Boorstrap folders to reduce the size of the NuGet cache dir"
101+
echo "== Deleting Out/frontend/Nuget/tmp, Out/bin, Out/Bootstrap folders to reduce the size of the NuGet cache dir"
95102
rm -rf Out/frontend/Nuget/tmp Out/Objects.noindex Out/bin Out/BootStrap
96103
echo "== Disk usage of folders in Out"
97104
du -sh Out/*
@@ -103,7 +110,7 @@ jobs:
103110
set -eu
104111
function run_build {
105112
# the disks on Azure Pipeline VMs are too small to build everything, so let's instead run tests
106-
bash bxl.sh --use-dev $(BxlCommonArgs) /logsDirectory:"Out/Logs/$(BxlMatrixName)" $(BxlMatrixArgs) "/f:tag='test'"
113+
bash bxl.sh --use-dev ${{ parameters.BxlCommonArgs }} /logsDirectory:"Out/Logs/$(BxlMatrixName)" $(BxlMatrixArgs) "/f:tag='test'"
107114
}
108115
109116
run_build || {

‎.azdo/linux/pipeline.yml

+83-10
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,17 @@ trigger:
44
include:
55
- master
66

7+
resources:
8+
repositories:
9+
- repository: 1esPipelines
10+
type: git
11+
name: 1ESPipelineTemplates/1ESPipelineTemplates
12+
ref: refs/tags/release
13+
14+
variables:
15+
PackageArtifactName: runtime.linux-x64.BuildXL.${{ parameters.LinuxRuntimePackageVersion }}
16+
PackageDir: $(Build.SourcesDirectory)/Public/Src/Sandbox/Linux/bin/$(PackageArtifactName)
17+
718
parameters:
819
- name: BuildSelfhost
920
type: boolean
@@ -16,13 +27,75 @@ parameters:
1627
default: false
1728
- name: LinuxRuntimePackageVersion
1829
type: string
19-
default: 0.1.0-$(Build.BuildNumber)
20-
21-
jobs:
22-
- ${{ if parameters.BuildSelfhost }}:
23-
- template: job-selfhost.yml
24-
- ${{ if parameters.BuildLinuxRuntime }}:
25-
- template: job-linux-runtime.yml
26-
parameters:
27-
PublishNuget: ${{ parameters.PublishLinuxRuntimeNuget }}
28-
PackageVersion: ${{ parameters.LinuxRuntimePackageVersion }}
30+
default: 0.1.0-$(Build.BuildNumber).$(System.JobAttempt)
31+
32+
extends:
33+
template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines
34+
parameters:
35+
36+
# Container in which to run *all* jobs (unfortunately 1ESPipelineTemplates allows only one container for all jobs)
37+
# Chosen because the Linux runtime binaries must be built for 'manylinux2014' (more precisely, against glibc 2.17);
38+
# currently, all other jobs in this pipeline opt out of containerization.
39+
container:
40+
image: quay.io/pypa/manylinux2014_x86_64
41+
42+
stages:
43+
- stage: Build
44+
jobs:
45+
# Build and test selfhost with BuildXL
46+
- ${{ if parameters.BuildSelfhost }}:
47+
- template: /.azdo/linux/job-selfhost.yml@self
48+
49+
# Build Linux native runtime libraries with make
50+
- ${{ if parameters.BuildLinuxRuntime }}:
51+
- job: BuildLinuxRuntime
52+
displayName: Build Linux Runtime
53+
pool:
54+
vmImage: ubuntu-latest
55+
templateContext:
56+
inputs:
57+
- input: checkout
58+
repository: self
59+
fetchDepth: 1
60+
steps:
61+
- bash: make cleanall && make all -j
62+
workingDirectory: Public/Src/Sandbox/Linux
63+
failOnStderr: true
64+
displayName: Build Native
65+
66+
- bash: bash mknuget.sh ${{ parameters.LinuxRuntimePackageVersion }}
67+
workingDirectory: Public/Src/Sandbox/Linux
68+
displayName: Pack Nuget
69+
70+
- task: PublishPipelineArtifact@1
71+
inputs:
72+
targetPath: $(PackageDir)
73+
artifactName: $(PackageArtifactName)
74+
condition: always()
75+
continueOnError: true
76+
displayName: Publish Pipeline Artifact
77+
78+
# Publish the built runtime in a separate job
79+
# TODO: this could be merged with the previous job if the container image had all the required tools preinstalled
80+
- ${{ if parameters.PublishLinuxRuntimeNuget }}:
81+
- job: PublishNativeRuntimeNuget
82+
displayName: Publish Linux Runtime NuGet
83+
dependsOn: [ BuildLinuxRuntime ]
84+
pool:
85+
vmImage: ubuntu-latest
86+
templateContext:
87+
breakGlass:
88+
justification: "Because 1ESPipelineTemplates allows only a single container for all jobs, and 'quay.io/pypa/manylinux2014_x86_64' does not have some tools needed for NuGet installed"
89+
steps:
90+
- checkout: none
91+
- task: DownloadPipelineArtifact@2
92+
inputs:
93+
artifact: $(PackageArtifactName)
94+
path: $(Build.ArtifactStagingDirectory)/$(PackageArtifactName)
95+
- template: /.azdo/linux/step-nuget-config.yml@self
96+
- bash: |
97+
set -eu
98+
zip -r ../$(PackageArtifactName).nupkg *
99+
dotnet nuget push --skip-duplicate --api-key "AzureDevOps" --source "$(Feed-BuildXL.Selfhost)" ../$(PackageArtifactName).nupkg
100+
displayName: Push NuGet
101+
workingDirectory: $(Build.ArtifactStagingDirectory)/$(PackageArtifactName)

‎.azdo/linux/step-install-mono.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
steps:
2+
- bash: |
3+
set -eu
4+
5+
# install mono
6+
sudo apt-get update
7+
sudo apt-get install -y mono-complete mono-devel
8+
mono --version
9+
displayName: Install Mono

‎.azdo/linux/step-nuget-config.yml

+10-11
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,16 @@ steps:
22
- bash: |
33
set -eu
44
5-
# install mono
6-
sudo apt-get update
7-
sudo apt-get install -y mono-complete mono-devel
8-
mono --version
9-
displayName: Install Prerequisites
5+
function addNugetSource {
6+
local source="$1"
7+
local name="$2"
8+
local password="$3"
109
11-
- bash: |
12-
set -eu
13-
mono Shared/Tools/NuGet.exe sources remove -Name "BuildXL.Selfhost" || true
14-
mono Shared/Tools/NuGet.exe sources remove -Name "BuildXL" || true
10+
dotnet nuget remove source "$name" 2>&1 > /dev/null || true
11+
echo "Adding '$name' -> '$source'"
12+
dotnet nuget add source "$source" --name "$name" --username tsebuild --password "$password" --store-password-in-clear-text
13+
}
1514
16-
mono Shared/Tools/NuGet.exe sources add -Name "BuildXL.Selfhost" -Source "$(Feed-BuildXL.Selfhost)" -username tsebuild -password $(PAT-TseBuild-AzureDevOps-CloudBuild)
17-
mono Shared/Tools/NuGet.exe sources add -Name "BuildXL" -Source "$(Feed-BuildXL)" -username tsebuild -password $(PAT-TseBuild-AzureDevOps-1esSharedAssets)
15+
addNugetSource "$(Feed-BuildXL.Selfhost)" "BuildXL.Selfhost" "$(PAT-TseBuild-AzureDevOps-CloudBuild)"
16+
addNugetSource "$(Feed-BuildXL)" "BuildXL" "$(PAT-TseBuild-AzureDevOps-1esSharedAssets)"
1817
displayName: Add NuGet Sources

‎Private/macOS/notarize.sh

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ fi
6262

6363
if [[ -z $arg_Password ]]; then
6464
echo "[INFO] Password not set; using the AC_PASSWORD entry from the keychain"
65+
# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]
6566
arg_Password="@keychain:AC_PASSWORD"
6667
fi
6768

‎Public/Src/Sandbox/Linux/mknuget.sh

+13-7
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
#!/bin/bash
22

3-
readonly __dir=$(cd `dirname ${BASH_SOURCE[0]}` && pwd)
4-
53
set -euo pipefail
64

5+
readonly __dir=$(cd `dirname ${BASH_SOURCE[0]}` && pwd)
76
readonly version="$1"
87
readonly pkgName="runtime.linux-x64.BuildXL.${version}"
8+
readonly binDir="${__dir}/bin"
99

10-
cd "${__dir}"
11-
make cleanall
12-
bash build-manylinux.sh
10+
if [[ ! -d "${binDir}/debug" ]] || [[ ! -d "${binDir}/release" ]]; then
11+
echo "One of '$binDir/{debug,release}' folders not found. Build the native runtime libraries first"
12+
exit 1
13+
fi
1314

14-
cd bin
15+
cd "${__dir}/bin"
1516

1617
rm -rf "${pkgName}"
1718
mkdir -p "${pkgName}/ref/netstandard"
@@ -38,4 +39,9 @@ cat > "${pkgName}/runtime.linux-x64.buildxl.nuspec" <<EOF
3839
EOF
3940

4041
cd "${pkgName}"
41-
zip -v ../${pkgName}.nupkg *
42+
if which zip > /dev/null 2>&1; then
43+
zip -r ../${pkgName}.nupkg *
44+
else
45+
pwd
46+
find . -type f
47+
fi

‎RunCheckInTestsWithPAT.ps1

+5-7
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,10 @@
1515
[Environment]::SetEnvironmentVariable("MSENG_GIT_PAT", $msEngGitPat, "Process")
1616
[Environment]::SetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH", $ncPath, "Process")
1717

18-
[Environment]::SetEnvironmentVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "
19-
{
20-
'endpointCredentials': [
21-
{'endpoint':'https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json', 'password':'$1esPat'},
22-
{'endpoint':'https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json', 'password':'$cbPat'}
23-
]
24-
}", "Process")
18+
[Environment]::SetEnvironmentVariable(
19+
"VSS_NUGET_EXTERNAL_FEED_ENDPOINTS",
20+
<#[SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]#>
21+
"{ 'endpointCredentials': [ {'endpoint':'https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json', 'password':'$1esPat'}, {'endpoint':'https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json', 'password':'$cbPat'} ]}",
22+
"Process")
2523

2624
.\RunCheckInTests.cmd /lab $args /internal

‎bxl.sh

+13-12
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,8 @@ function installCredProvider() {
198198
fi
199199

200200
# Download the artifacts credential provider
201-
wget -q -c https://github.com/microsoft/artifacts-credprovider/releases/download/v1.0.0/Microsoft.NuGet.CredentialProvider.tar.gz -O - | tar -xz -C $destinationFolder
201+
mkdir -p "$destinationFolder"
202+
wget -q -c https://github.com/microsoft/artifacts-credprovider/releases/download/v1.0.0/Microsoft.NuGet.CredentialProvider.tar.gz -O - | tar -xz -C "$destinationFolder"
202203

203204
# Remove the .exe, since we want to replace it with a script that runs on Mac/Linux
204205
rm "$credentialProviderExe"
@@ -250,17 +251,6 @@ if [[ -n "$arg_Cgmanifest" ]]; then
250251
arg_Positional+=(/generateCgManifestForNugets:"${MY_DIR}/cg/nuget/cgmanifest.json")
251252
fi
252253

253-
if [[ -n "$arg_UseDev" ]]; then
254-
if [[ ! -f $MY_DIR/Out/Selfhost/Dev/bxl ]]; then
255-
print_error "Error: Could not find the dev deployment. Make sure you build with --deploy-dev first."
256-
exit 1
257-
fi
258-
259-
export BUILDXL_BIN=$MY_DIR/Out/Selfhost/Dev
260-
elif [[ -z "$BUILDXL_BIN" ]]; then
261-
getLkg
262-
fi
263-
264254
# if the nuget credential provider is not configured (and the build is an internal one, which is where it is needed)
265255
# download and install the artifacts credential provider
266256
if [[ -n "$arg_Internal" ]] && [[ ! -d "${NUGET_CREDENTIALPROVIDERS_PATH:-}" ]]; then
@@ -305,6 +295,17 @@ if [[ -z "$TF_BUILD" ]];then
305295
arg_Positional+=("/p:[Sdk.BuildXL]validateLinuxRuntime=0")
306296
fi
307297

298+
if [[ -n "$arg_UseDev" ]]; then
299+
if [[ ! -f $MY_DIR/Out/Selfhost/Dev/bxl ]]; then
300+
print_error "Error: Could not find the dev deployment. Make sure you build with --deploy-dev first."
301+
exit 1
302+
fi
303+
304+
export BUILDXL_BIN=$MY_DIR/Out/Selfhost/Dev
305+
elif [[ -z "$BUILDXL_BIN" ]]; then
306+
getLkg
307+
fi
308+
308309
compileWithBxl ${arg_Positional[@]}
309310

310311
if [[ -n "$arg_DeployDev" ]]; then

0 commit comments

Comments
 (0)
Please sign in to comment.