From 11b414a0404904e31e60904b2ea3f3bdd90686f6 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:35:27 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 6919 (#38942) * Add autorest-preview pipeline * Add emitternpminstall * Always install * Use shorter leg name * Add short circuiting to emitter install * Use language matrix function * Don't look for subfolders in the matrix package folders * Revert unnecessary eng/common changes * Rewrite GetPullRequestUrl to Get-BuildSourceDescription * Remove alias from Invoke-LoggedCommand * Use invoke-expression instead of shelling out * Add better job splitting * Replace Folder with Directory --------- Co-authored-by: Patrick Hallisey --- .../templates/steps/sparse-checkout.yml | 11 +- .../scripts/Get-BuildSourceDescription.ps1 | 24 +++++ .../Helpers/CommandInvocation-Helpers.ps1 | 42 ++++++++ eng/common/scripts/New-RegenerateMatrix.ps1 | 102 ++++++++++++++++++ .../scripts/TypeSpec-Project-Generate.ps1 | 23 ++-- eng/common/scripts/Update-GeneratedSdks.ps1 | 16 +++ eng/common/scripts/common.ps1 | 4 +- 7 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 eng/common/scripts/Get-BuildSourceDescription.ps1 create mode 100644 eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 create mode 100644 eng/common/scripts/New-RegenerateMatrix.ps1 create mode 100644 eng/common/scripts/Update-GeneratedSdks.ps1 diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index ab95453954cbe..448cb2c2e313a 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -29,7 +29,7 @@ steps: if (!$dir) { $dir = "./$($repository.Name)" } - New-Item $dir -ItemType Directory -Force + New-Item $dir -ItemType Directory -Force | Out-Null Push-Location $dir if (Test-Path .git/info/sparse-checkout) { @@ -70,9 +70,14 @@ steps: # sparse-checkout commands after initial checkout will auto-checkout again if (!$hasInitialized) { - Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" + # Remove refs/heads/ prefix from branch names + $commitish = $repository.Commitish -replace '^refs/heads/', '' + + # use -- to prevent git from interpreting the commitish as a path + Write-Host "git -c advice.detachedHead=false checkout $commitish --" + # This will use the default branch if repo.Commitish is empty - git -c advice.detachedHead=false checkout $($repository.Commitish) + git -c advice.detachedHead=false checkout $commitish -- } else { Write-Host "Skipping checkout as repo has already been initialized" } diff --git a/eng/common/scripts/Get-BuildSourceDescription.ps1 b/eng/common/scripts/Get-BuildSourceDescription.ps1 new file mode 100644 index 0000000000000..b0856101538d8 --- /dev/null +++ b/eng/common/scripts/Get-BuildSourceDescription.ps1 @@ -0,0 +1,24 @@ +param( + [string]$Variable, + [switch]$IsOutput +) + +$repoUrl = $env:BUILD_REPOSITORY_URI +$sourceBranch = $env:BUILD_SOURCEBRANCH + +$description = "[$sourceBranch]($repoUrl/tree/$sourceBranch)" +if ($sourceBranch -match "^refs/heads/(.+)$") { + $description = "Branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" +} elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $description = "Tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" +} elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $description = "Pull request: $repoUrl/pull/$($Matches[1])" +} + +if ($IsOutput) { + Write-Host "Setting output variable '$Variable' to '$description'" + Write-Host "##vso[task.setvariable variable=$Variable;isoutput=true]$description" +} else { + Write-Host "Setting variable '$Variable' to '$description'" + Write-Host "##vso[task.setvariable variable=$Variable]$description" +} diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 new file mode 100644 index 0000000000000..5dc0c8c7da1af --- /dev/null +++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 @@ -0,0 +1,42 @@ +function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) +{ + $pipelineBuild = !!$env:TF_BUILD + $startTime = Get-Date + + if($pipelineBuild -and $GroupOutput) { + Write-Host "##[group]$Command" + } else { + Write-Host "> $Command" + } + + if($ExecutePath) { + Push-Location $ExecutePath + } + + try { + Invoke-Expression $Command + + $duration = (Get-Date) - $startTime + + if($pipelineBuild -and $GroupOutput) { + Write-Host "##[endgroup]" + } + + if($LastExitCode -ne 0) + { + if($pipelineBuild) { + Write-Error "##[error]Command failed to execute ($duration): $Command`n" + } else { + Write-Error "Command failed to execute ($duration): $Command`n" + } + } + else { + Write-Host "Command succeeded ($duration)`n" + } + } + finally { + if($ExecutePath) { + Pop-Location + } + } +} diff --git a/eng/common/scripts/New-RegenerateMatrix.ps1 b/eng/common/scripts/New-RegenerateMatrix.ps1 new file mode 100644 index 0000000000000..1df97420c25d8 --- /dev/null +++ b/eng/common/scripts/New-RegenerateMatrix.ps1 @@ -0,0 +1,102 @@ +[CmdLetBinding()] +param ( + [Parameter()] + [string]$OutputDirectory, + + [Parameter()] + [string]$OutputVariableName, + + [Parameter()] + [int]$JobCount = 8, + + # The minimum number of items per job. If the number of items is less than this, then the number of jobs will be reduced. + [Parameter()] + [int]$MinimumPerJob = 10, + + [Parameter()] + [string]$OnlyTypespec +) + +. (Join-Path $PSScriptRoot common.ps1) + +[bool]$OnlyTypespec = $OnlyTypespec -in @("true", "t", "1", "yes", "y") + +# Divide the items into groups of approximately equal size. +function Split-Items([array]$Items) { + # given $Items.Length = 22 and $JobCount = 5 + # then $itemsPerGroup = 4 + # and $largeJobCount = 2 + # and $group.Length = 5, 5, 4, 4, 4 + $itemCount = $Items.Length + $jobsForMinimum = $itemCount -lt $MinimumPerJob ? 1 : [math]::Floor($itemCount / $MinimumPerJob) + + if ($JobCount -gt $jobsForMinimum) { + $JobCount = $jobsForMinimum + } + + $itemsPerGroup = [math]::Floor($itemCount / $JobCount) + $largeJobCount = $itemCount % $itemsPerGroup + $groups = [object[]]::new($JobCount) + + $i = 0 + for ($g = 0; $g -lt $JobCount; $g++) { + $groupLength = if ($g -lt $largeJobCount) { $itemsPerGroup + 1 } else { $itemsPerGroup } + $group = [object[]]::new($groupLength) + $groups[$g] = $group + for ($gi = 0; $gi -lt $groupLength; $gi++) { + $group[$gi] = $Items[$i++] + } + } + + Write-Host "$itemCount items split into $JobCount groups of approximately $itemsPerGroup items each." + + return , $groups +} + +# ensure the output directory exists +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + +if (Test-Path "Function:$GetDirectoriesForGenerationFn") { + $directoriesForGeneration = &$GetDirectoriesForGenerationFn +} +else { + $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory +} + +if ($OnlyTypespec) { + $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } +} + +[array]$packageDirectories = $directoriesForGeneration +| Sort-Object -Property FullName +| ForEach-Object { + [ordered]@{ + "PackageDirectory" = "$($_.Parent.Name)/$($_.Name)" + "ServiceArea" = $_.Parent.Name + } +} + +$batches = Split-Items -Items $packageDirectories + +$matrix = [ordered]@{} +for ($i = 0; $i -lt $batches.Length; $i++) { + $batch = $batches[$i] + $json = $batch.PackageDirectory | ConvertTo-Json -AsArray + + $firstPrefix = $batch[0].ServiceArea.Substring(0, 2) + $lastPrefix = $batch[-1].ServiceArea.Substring(0, 2) + + $key = "$firstPrefix`_$lastPrefix`_$i" + $fileName = "$key.json" + + Write-Host "`n`n==================================" + Write-Host $fileName + Write-Host "==================================" + $json | Out-Host + $json | Out-File "$OutputDirectory/$fileName" + + $matrix[$key] = [ordered]@{ "JobKey" = $key; "DirectoryList" = $fileName } +} + +$compressed = ConvertTo-Json $matrix -Depth 100 -Compress +Write-Output "##vso[task.setVariable variable=$OutputVariableName;isOutput=true]$compressed" diff --git a/eng/common/scripts/TypeSpec-Project-Generate.ps1 b/eng/common/scripts/TypeSpec-Project-Generate.ps1 index e323f42aff6dc..05a0e0bdfd45d 100644 --- a/eng/common/scripts/TypeSpec-Project-Generate.ps1 +++ b/eng/common/scripts/TypeSpec-Project-Generate.ps1 @@ -11,6 +11,7 @@ param ( $ErrorActionPreference = "Stop" . $PSScriptRoot/Helpers/PSModule-Helpers.ps1 +. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1 . $PSScriptRoot/common.ps1 Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module @@ -21,38 +22,30 @@ function NpmInstallForProject([string]$workingDirectory) { Write-Host "Generating from $currentDur" if (Test-Path "package.json") { + Write-Host "Removing existing package.json" Remove-Item -Path "package.json" -Force } if (Test-Path ".npmrc") { + Write-Host "Removing existing .nprc" Remove-Item -Path ".npmrc" -Force } if (Test-Path "node_modules") { + Write-Host "Removing existing node_modules" Remove-Item -Path "node_modules" -Force -Recurse } if (Test-Path "package-lock.json") { + Write-Host "Removing existing package-lock.json" Remove-Item -Path "package-lock.json" -Force } - #default to root/eng/emitter-package.json but you can override by writing - #Get-${Language}-EmitterPackageJsonPath in your Language-Settings.ps1 $replacementPackageJson = Join-Path $PSScriptRoot "../../emitter-package.json" - if (Test-Path "Function:$GetEmitterPackageJsonPathFn") { - $replacementPackageJson = &$GetEmitterPackageJsonPathFn - } Write-Host("Copying package.json from $replacementPackageJson") Copy-Item -Path $replacementPackageJson -Destination "package.json" -Force - - #default to root/eng/emitter-package-lock.json but you can override by writing - #Get-${Language}-EmitterPackageLockPath in your Language-Settings.ps1 $emitterPackageLock = Join-Path $PSScriptRoot "../../emitter-package-lock.json" - if (Test-Path "Function:$GetEmitterPackageLockPathFn") { - $emitterPackageLock = &$GetEmitterPackageLockPathFn - } - $usingLockFile = Test-Path $emitterPackageLock if ($usingLockFile) { @@ -68,12 +61,10 @@ function NpmInstallForProject([string]$workingDirectory) { } if ($usingLockFile) { - Write-Host "> npm ci" - npm ci + Invoke-LoggedCommand "npm ci" } else { - Write-Host "> npm install" - npm install + Invoke-LoggedCommand "npm install" } if ($LASTEXITCODE) { exit $LASTEXITCODE } diff --git a/eng/common/scripts/Update-GeneratedSdks.ps1 b/eng/common/scripts/Update-GeneratedSdks.ps1 new file mode 100644 index 0000000000000..dd671f6d8ad83 --- /dev/null +++ b/eng/common/scripts/Update-GeneratedSdks.ps1 @@ -0,0 +1,16 @@ +[CmdLetBinding()] +param( + [Parameter(Mandatory)] + [string]$PackageDirectoriesFile +) + +. $PSScriptRoot/common.ps1 +. $PSScriptRoot/Helpers/CommandInvocation-Helpers.ps1 + +$ErrorActionPreference = 'Stop' + +if (Test-Path "Function:$UpdateGeneratedSdksFn") { + &$UpdateGeneratedSdksFn $PackageDirectoriesFile +} else { + Write-Error "Function $UpdateGeneratedSdksFn not implemented in Language-Settings.ps1" +} diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 39d65cdd68183..cef0b23c5620a 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -60,8 +60,8 @@ $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme" $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink" $GetEmitterAdditionalOptionsFn = "Get-${Language}-EmitterAdditionalOptions" $GetEmitterNameFn = "Get-${Language}-EmitterName" -$GetEmitterPackageJsonPathFn = "Get-${Language}-EmitterPackageJsonPath" -$GetEmitterPackageLockPathFn = "Get-${Language}-EmitterPackageLockPath" +$GetDirectoriesForGenerationFn = "Get-${Language}-DirectoriesForGeneration" +$UpdateGeneratedSdksFn = "Update-${Language}-GeneratedSdks" # Expected to be set in eng/scripts/docs/Docs-Onboarding.ps1 $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"