[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -67,25 +67,25 @@ Note: if the Uri is a new place, you will need to add a subscription from that p
       <SourceBuildId>5596</SourceBuildId>
     </Dependency>
     <!-- Arcade -->
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20224.11">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>ff09a8a065a615c2fde8df82106bcd1f0fcb32aa</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenFacades" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.GenFacades" Version="5.0.0-beta.20224.11">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>ff09a8a065a615c2fde8df82106bcd1f0fcb32aa</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20224.11">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>ff09a8a065a615c2fde8df82106bcd1f0fcb32aa</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="5.0.0-beta.20224.11">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>ff09a8a065a615c2fde8df82106bcd1f0fcb32aa</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.XUnitExtensions" Version="5.0.0-beta.20224.11">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>ff09a8a065a615c2fde8df82106bcd1f0fcb32aa</Sha>
     </Dependency>
     <Dependency Name="runtime.win-x64.Microsoft.NETCore.ILAsm" Version="5.0.0-preview.5.20223.4">
       <Uri>https://github.com/dotnet/runtime</Uri>

eng/Versions.props

@@ -39,9 +39,9 @@
   </PropertyGroup>
   <!-- Arcade -->
   <PropertyGroup>
-    <MicrosoftDotNetGenFacadesPackageVersion>5.0.0-beta.20201.2</MicrosoftDotNetGenFacadesPackageVersion>
-    <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.20201.2</MicrosoftDotNetRemoteExecutorVersion>
-    <MicrosoftDotNetXUnitExtensionsPackageVersion>5.0.0-beta.20201.2</MicrosoftDotNetXUnitExtensionsPackageVersion>
+    <MicrosoftDotNetGenFacadesPackageVersion>5.0.0-beta.20224.11</MicrosoftDotNetGenFacadesPackageVersion>
+    <MicrosoftDotNetRemoteExecutorVersion>5.0.0-beta.20224.11</MicrosoftDotNetRemoteExecutorVersion>
+    <MicrosoftDotNetXUnitExtensionsPackageVersion>5.0.0-beta.20224.11</MicrosoftDotNetXUnitExtensionsPackageVersion>
     <MicrosoftNETTestSdkVersion>16.5.0</MicrosoftNETTestSdkVersion>
   </PropertyGroup>
   <!-- Below have no corresponding entries in Versions.Details.XML because they are not updated via Maestro -->

eng/common/SetupNugetSources.ps1

@@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) {
     $passwordElement.SetAttribute("value", $Password)
 }
 
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
     $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
 
     Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
@@ -123,19 +123,21 @@ if ($creds -eq $null) {
     $doc.DocumentElement.AppendChild($creds) | Out-Null
 }
 
+$userName = "dn-bot"
+
 # Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
 
 $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
 if ($dotnet3Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
 $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
 if ($dotnet31Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
 $doc.Save($filename)

eng/common/build.ps1

@@ -69,9 +69,6 @@ function Print-Usage() {
 
 . $PSScriptRoot\tools.ps1
 
-$moduleLocation = Resolve-Path "$PSScriptRoot\..\Screenshots.win.psm1"
-$initScreenshotsModule = [scriptblock]::Create("Import-Module $moduleLocation")
-
 function InitializeCustomToolset {
   if (-not $restore) {
     return
@@ -145,39 +142,12 @@ try {
     InitializeNativeTools
   }
 
-  $TakeScreenshots = $ci -and ($test -or $integrationTest -or $performanceTest);
-  $ImageLogs = '';
-  if ($TakeScreenshots) {
-    $ImageLogs = Join-Path $LogDir 'screenshots'
-    Create-Directory $ImageLogs
-
-    [ScriptBlock] $ScreenshotCaptureScript = {
-      param($ImageLogs)
-      Start-CaptureScreenshots "$ImageLogs"
-    };
-
-    $job = Start-Job -InitializationScript $initScreenshotsModule `
-              -ScriptBlock $ScreenshotCaptureScript `
-              -ArgumentList $ImageLogs
-  }
-
   Build
 }
 catch {
   Write-Host $_.ScriptStackTrace
   Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
   ExitWithExitCode 1
 }
-finally {
-  if ($TakeScreenshots) {
-    [ScriptBlock] $ScreenshotCaptureScript = {
-      param($ImageLogs)
-      Stop-CaptureScreenshots -TargetDir $ImageLogs
-    };
-    Start-Job -InitializationScript $initScreenshotsModule `
-              -ScriptBlock $ScreenshotCaptureScript `
-              -ArgumentList $ImageLogs | Receive-Job -AutoRemoveJob -Wait
-  }
-}
 
 ExitWithExitCode 0

eng/common/cross/build-rootfs.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 usage()
 {
     echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
@@ -15,6 +17,8 @@ __CodeName=xenial
 __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 __InitialDir=$PWD
 __BuildArch=arm
+__AlpineArch=armv7
+__QEMUArch=arm
 __UbuntuArch=armhf
 __UbuntuRepo="http://ports.ubuntu.com/"
 __LLDB_Package="liblldb-3.9-dev"
@@ -26,9 +30,10 @@ __UbuntuPackages="build-essential"
 __AlpinePackages="alpine-base"
 __AlpinePackages+=" build-base"
 __AlpinePackages+=" linux-headers"
-__AlpinePackagesEdgeTesting=" lldb-dev"
-__AlpinePackagesEdgeMain=" llvm9-libs"
+__AlpinePackagesEdgeCommunity=" lldb-dev"
+__AlpinePackagesEdgeMain=" llvm10-libs"
 __AlpinePackagesEdgeMain+=" python3"
+__AlpinePackagesEdgeMain+=" libedit"
 
 # symlinks fixer
 __UbuntuPackages+=" symlinks"
@@ -55,12 +60,11 @@ __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
 
 __FreeBSDBase="12.1-RELEASE"
-__FreeBSDPkg="1.10.5"
+__FreeBSDPkg="1.12.0"
 __FreeBSDPackages="libunwind"
 __FreeBSDPackages+=" icu"
 __FreeBSDPackages+=" libinotify"
 __FreeBSDPackages+=" lttng-ust"
-__FreeBSDPackages+=" llvm-90"
 __FreeBSDPackages+=" krb5"
 
 __UnprocessedBuildArgs=
@@ -78,7 +82,7 @@ while :; do
         arm)
             __BuildArch=arm
             __UbuntuArch=armhf
-            __AlpineArch=armhf
+            __AlpineArch=armv7
             __QEMUArch=arm
             ;;
         arm64)
@@ -205,7 +209,7 @@ fi
 
 if [ -d "$__RootfsDir" ]; then
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
     rm -rf $__RootfsDir
 fi
@@ -231,9 +235,9 @@ if [[ "$__CodeName" == "alpine" ]]; then
       add $__AlpinePackagesEdgeMain
 
     $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-      -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
+      -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
       -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
-      add $__AlpinePackagesEdgeTesting
+      add $__AlpinePackagesEdgeCommunity
 
     rm -r $__ApkToolsDir
 elif [[ "$__CodeName" == "freebsd" ]]; then
@@ -246,7 +250,9 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
     # get and build package manager
     wget -O -  https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz  |  tar -C $__RootfsDir/tmp -zxf -
     cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
-    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install
+    # needed for install to succeed
+    mkdir -p $__RootfsDir/host/etc
+    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install
     rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
     # install packages we need.
     $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
@@ -260,7 +266,7 @@ elif [[ -n $__CodeName ]]; then
     chroot $__RootfsDir symlinks -cr /usr
 
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
 
     if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then

eng/common/post-build/check-channel-consistency.ps1

@@ -6,6 +6,11 @@ param(
 try {
   . $PSScriptRoot\post-build-utils.ps1
 
+  if ($PromoteToChannels -eq "") {
+    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+    ExitWithExitCode 0
+  }
+
   # Check that every channel that Maestro told to promote the build to 
   # is available in YAML
   $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ }

eng/common/post-build/symbols-validation.ps1

@@ -1,7 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion     # Version of dotnet symbol to use
+  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion,    # Version of dotnet symbol to use
+  [Parameter(Mandatory=$false)][switch] $ContinueOnError        # If we should keep checking symbols after an error
 )
 
 function FirstMatchingSymbolDescriptionOrDefault {
@@ -125,6 +126,8 @@ function CheckSymbolsAvailable {
     Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
   }
 
+  $TotalFailures = 0
+
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {
       $FileName = $_.Name
@@ -148,11 +151,22 @@ function CheckSymbolsAvailable {
 
       if ($Status -ne 0) {
         Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
-        ExitWithExitCode $exitCode
+
+        if ($ContinueOnError) {
+          $TotalFailures++
+        }
+        else {
+          ExitWithExitCode 1
+        }
       }
 
       Write-Host
     }
+
+  if ($TotalFailures -ne 0) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
+    ExitWithExitCode 1
+  }
 }
 
 function InstallDotnetSymbol {

eng/common/sdk-task.ps1

@@ -57,6 +57,18 @@ try {
     ExitWithExitCode 1
   }
 
+  if( $msbuildEngine -eq "vs") {
+    # Ensure desktop MSBuild is available for sdk tasks.
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) {
+      $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty
+    }
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty
+    }
+
+    InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+  }
+
   $taskProject = GetSdkTaskProject $task
   if (!(Test-Path $taskProject)) {
     Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red

eng/common/sdl/extract-artifact-packages.ps1

@@ -63,7 +63,7 @@ try {
           }
     }
     catch {
-      Write-Host $_.ScriptStackTrace
+      Write-Host $_
       Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
       ExitWithExitCode 1
     }
@@ -74,7 +74,7 @@ try {
   Measure-Command { ExtractArtifacts }
 }
 catch {
-  Write-Host $_.ScriptStackTrace
+  Write-Host $_
   Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
   ExitWithExitCode 1
 }

eng/common/sdl/init-sdl.ps1

@@ -24,7 +24,7 @@ $ProgressPreference = 'SilentlyContinue'
 # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
 $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
 $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
-$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1"
+$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
 $zipFile = "$WorkingDirectory/gdn.zip"
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.7.2"/>
+  <package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/>
 </packages>

eng/common/templates/job/execute-sdl.yml

@@ -6,6 +6,7 @@ parameters:
   # 'continueOnError', the parameter value is not correctly picked up.
   # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
   sdlContinueOnError: false                                    # optional: determines whether to continue the build if the step errors;
+  downloadArtifacts: true                                      # optional: determines if the artifacts should be dowloaded
   dependsOn: ''                                                # Optional: dependencies of the job
   artifactNames: ''                                            # Optional: patterns supplied to DownloadBuildArtifacts
                                                                # Usage:
@@ -31,8 +32,20 @@ jobs:
   steps:
   - checkout: self
     clean: true
-  - ${{ if ne(parameters.artifactNames, '') }}:
-    - ${{ each artifactName in parameters.artifactNames }}:
+  - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
+    - ${{ if ne(parameters.artifactNames, '') }}:
+      - ${{ each artifactName in parameters.artifactNames }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Build Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: ${{ artifactName }}
+            downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+    - ${{ if eq(parameters.artifactNames, '') }}:
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Artifacts
         inputs:
@@ -41,20 +54,9 @@ jobs:
           project: $(AzDOProjectName)
           pipeline: $(AzDOPipelineId)
           buildId: $(AzDOBuildId)
-          artifactName: ${{ artifactName }}
+          downloadType: specific files
+          itemPattern: "**"
           downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
-  - ${{ if eq(parameters.artifactNames, '') }}:
-    - task: DownloadBuildArtifacts@0
-      displayName: Download Build Artifacts
-      inputs:
-        buildType: specific
-        buildVersionToDownload: specific
-        project: $(AzDOProjectName)
-        pipeline: $(AzDOPipelineId)
-        buildId: $(AzDOBuildId)
-        downloadType: specific files
-        itemPattern: "**"
-        downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@@ -81,7 +83,7 @@ jobs:
       continueOnError: ${{ parameters.sdlContinueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
     - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.0.7.2
+        -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}