From 21cbee17d36501664312af4de087edfb0e6ab51e Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 11 Dec 2024 10:24:35 -0800 Subject: [PATCH 01/15] Consolidate artifacts --- .../official/jobs/build-artifacts-linux.yml | 22 +++++----- .../official/jobs/build-artifacts-windows.yml | 40 +++++++++---------- eng/ci/templates/steps/build-site-ext.yml | 12 ++++-- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index ef6cf4b728..283cd58a2d 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -6,24 +6,24 @@ jobs: project: src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj configuration: release runtime: linux-x64 - artifacts_path: $(Build.ArtifactStagingDirectory) - log_dir: $(artifacts_path)/log - zip_artifacts_path: $(artifacts_path)/Linux + drop_path: $(Build.ArtifactStagingDirectory) + log_path: $(drop_path)/logs + linux_drop_path: $(drop_path)/linux build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true' templateContext: - outputParentDirectory: $(artifacts_path) + outputParentDirectory: $(drop_path) outputs: # TODO: onboard to Azure Artifacts Drops to allow accessing this from docker linux pipeline in msazure # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/azure-artifacts/artifact-services-onboarding - output: pipelineArtifact displayName: Publish linux artifacts - path: $(zip_artifacts_path) - artifact: Linux + path: $(linux_drop_path) + artifact: drop_linux - output: pipelineArtifact displayName: Publish logs - path: $(log_dir) - artifact: Linux_Log + path: $(log_path) + artifact: logs_linux sbomEnabled: false condition: always() @@ -41,7 +41,7 @@ jobs: command: custom custom: restore projects: $(project) - arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 + arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_path)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 - task: DotNetCoreCLI@2 displayName: Build @@ -49,7 +49,7 @@ jobs: command: custom custom: build projects: $(project) - arguments: '$(build_args) --no-restore -bl:$(log_dir)/build.binlog' + arguments: '$(build_args) --no-restore -bl:$(log_path)/build.binlog' - task: DotNetCoreCLI@2 displayName: Publish @@ -59,4 +59,4 @@ jobs: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: $(project) - arguments: '$(build_args) --no-build -p:ZipArtifactsPath=$(zip_artifacts_path) -bl:$(log_dir)/publish.binlog' + arguments: '$(build_args) --no-build -bl:$(log_path)/publish.binlog -o $(linux_drop_path)/host' diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 7b3252da87..d8eb8cb82f 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -8,45 +8,41 @@ jobs: os: windows variables: - nuget_package_path: $(Build.ArtifactStagingDirectory)/NugetPackages - log_dir: $(Build.ArtifactStagingDirectory)/log + artifacts_path: $(Build.ArtifactStagingDirectory) + nuget_drop_path: $(artifacts_path)/nuget + windows_drop_path: $(artifacts_path)/windows + log_path: $(artifacts_path)/log templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputParentDirectory: $(artifacts_path) outputs: - output: pipelineArtifact displayName: Publish site extension - path: $(Build.ArtifactStagingDirectory)/SiteExtension - artifact: SiteExtension - - output: pipelineArtifact - displayName: Publish private site extension - path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension - artifact: PrivateSiteExtension - - output: pipelineArtifact - displayName: Publish site extension symbols - path: $(Build.ArtifactStagingDirectory)/SiteExtensionSymbols - artifact: Symbols + path: $(windows_drop_path) + artifact: drop_windows - output: pipelineArtifact displayName: Publish nuget packages - path: $(Build.ArtifactStagingDirectory)/NugetPackages - artifact: NugetPackages + path: $(nuget_drop_path) + artifact: drop_nuget - output: pipelineArtifact displayName: Publish logs - path: $(log_dir) - artifact: Windows_Log + path: $(log_path) + artifact: logs_windows sbomEnabled: false condition: always() steps: - template: /eng/ci/templates/install-dotnet.yml@self - template: /eng/ci/templates/steps/build-site-ext.yml@self + parameters: + publishDir: $(windows_drop_path) - task: DotNetCoreCLI@2 displayName: Build host packages inputs: command: custom custom: pack - arguments: -c release -o $(nuget_package_path) + arguments: -c release -o $(nuget_drop_path) projects: | **/WebJobs.Script.csproj **/WebJobs.Script.WebHost.csproj @@ -73,7 +69,7 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release -o $(nuget_package_path)' + arguments: '--no-build -c release -o $(nuget_drop_path)' projects: | **/WebJobs.Script.Abstractions.csproj @@ -89,14 +85,14 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release -o $(nuget_package_path)' + arguments: '--no-build -c release -o $(nuget_drop_path)' projects: | **/ExtensionsMetadataGenerator.csproj - template: ci/sign-files.yml@eng parameters: displayName: Sign NugetPackages - folderPath: $(nuget_package_path) + folderPath: $(nuget_drop_path) pattern: '*.nupkg' signType: nuget @@ -108,4 +104,4 @@ jobs: - task: DeleteFiles@1 displayName: Delete CodeSignSummary files inputs: - contents: '$(nuget_package_path)/**/CodeSignSummary-*.md' + contents: '$(nuget_drop_path)/**/CodeSignSummary-*.md' diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index a3c7c1d65a..eff8d581bc 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -2,6 +2,9 @@ parameters: - name: project type: string default: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +- name: publishDir + type: string + default: '' steps: # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' @@ -11,7 +14,7 @@ steps: command: custom custom: restore projects: ${{ parameters.project }} - arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog' + arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_path)/site_ext.restore.binlog' - task: DotNetCoreCLI@2 displayName: Build site extension @@ -19,7 +22,7 @@ steps: command: custom custom: build projects: ${{ parameters.project }} - arguments: '--no-restore -v m -c release -bl:$(log_dir)/site_ext.build.binlog' + arguments: '--no-restore -v m -c release -bl:$(log_path)/site_ext.build.binlog' - task: DotNetCoreCLI@2 displayName: Publish site extension @@ -29,4 +32,7 @@ steps: publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic projects: ${{ parameters.project }} - arguments: '--no-build -v m -c release -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' + ${{ if eq(parameters.publishDir, '') }}: + arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog' + ${{ else }}: + arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog -o ${{ parameters.publishDir }}' From 5f177b4ff69abafa43571c1455c026cda880c2ce Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 11 Dec 2024 11:52:56 -0800 Subject: [PATCH 02/15] Disable sdl spotBugs --- eng/ci/official-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 1455ec986c..640808112d 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -48,6 +48,8 @@ extends: codeql: # Move codeql for source languages to source analysis stage runSourceLanguagesInSourceAnalysis: true + spotBugs: + enabled: false # Explicitly disabling as our JavaWorker package causes this to light up and subsequently fail. stages: - stage: Build From 6e76b98ac720dedd52285b191c251b26731e5714 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 11 Dec 2024 16:24:24 -0800 Subject: [PATCH 03/15] Remove PrivateSiteExtension --- .../Publish.SingleTFM.targets | 36 +------------------ .../WebJobs.Script.SiteExtension.csproj | 4 +-- 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index 7137585c65..ea7b2fe591 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -10,10 +10,8 @@ ValidatePublishSettings; PublishProjectReferences; - PublishPrivateProjectReferences; RemoveUnneededRuntimes; MoveSymbols; - DeletePrivateSymbols; WriteHardLinkHashes; ZipAfterPublish;ZipArtifactsPath @@ -28,9 +26,7 @@ SiteExtension/$(Version)/ - $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - PrivateSiteExtension/SiteExtensions/Functions/ - $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) + $([MSBuild]::NormalizePath($([System.IO.Path]::Combine($(PublishDir), $(SiteExtensionRelativeDir))))) @@ -58,8 +54,6 @@ %(PublishRuntimeIdentifier.Identity) $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - %(PublishRuntimeIdentifier.PrivateExtension) false @@ -84,31 +78,11 @@ RemoveProperties="$(PublishPropertiesToRemove)"/> - - - - <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> - %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true - - - - - - - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/linux')" Condition="%(PrivateExtension)" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/osx')" Condition="%(PrivateExtension)" /> @@ -136,14 +110,6 @@ - - - - <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> - - - - diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index b1e34b916c..02c32a7fe3 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -18,8 +18,8 @@ - - + + From 25922b3dd16eccb427394b44eb97da52ccffe08f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 12 Dec 2024 09:03:25 -0800 Subject: [PATCH 04/15] stage release_notes.md as artifact --- .../official/jobs/build-artifacts-windows.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index d8eb8cb82f..471ecb40ea 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -8,13 +8,13 @@ jobs: os: windows variables: - artifacts_path: $(Build.ArtifactStagingDirectory) - nuget_drop_path: $(artifacts_path)/nuget - windows_drop_path: $(artifacts_path)/windows - log_path: $(artifacts_path)/log + drop_path: $(Build.ArtifactStagingDirectory) + nuget_drop_path: $(drop_path)/nuget + windows_drop_path: $(drop_path)/windows + log_path: $(drop_path)/log templateContext: - outputParentDirectory: $(artifacts_path) + outputParentDirectory: $(drop_path) outputs: - output: pipelineArtifact displayName: Publish site extension @@ -96,6 +96,13 @@ jobs: pattern: '*.nupkg' signType: nuget + - task: CopyFiles@2 + displayName: Stage Release Notes + inputs: + SourceFolder: $(Build.SourcesDirectory) + Contents: release_notes.md + TargetFolder: $(windows_drop_path) + - task: DeleteFiles@1 displayName: Delete CodeSignSummary files inputs: From 500063b2a7f499d917891410f038cc8dd20db09d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 12 Dec 2024 10:50:36 -0800 Subject: [PATCH 05/15] Add script to generate private site extension --- .../New-PrivateSiteExtension.ps1 | 99 +++++++++++++++++++ .../Publish.SingleTFM.targets | 5 +- .../WebJobs.Script.SiteExtension.csproj | 4 + 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 new file mode 100644 index 0000000000..2f33a85759 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -0,0 +1,99 @@ +<# + .SYNOPSIS + Produces a private site extension. + + .DESCRIPTION + Takes in a published site extension and produces a private site extension. + + .PARAMETER InputPath + The path of the published 'SiteExtension'. Default is "./SiteExtension". + + .PARAMETER OutputPath + The path to produce the private site extension to. When zipping, this is the name of the zip file. Default is "./PrivateSiteExtension". + + .PARAMETER Zip + [Switch] Include to produce site extension as a zip. + + .PARAMETER Force + [Switch] Include to overwrite existing files. + + .INPUTS + None. You can't pipe objects to Update-Month.ps1. + + .OUTPUTS + None. Update-Month.ps1 doesn't generate any output. +#> + +param ( + [string] $InputPath = "./SiteExtension", + [string] $OutputPath = "./PrivateSiteExtension", + [switch] $Zip, + [switch] $Force +) + +if (-not (Join-Path $InputPath "extension.xml" | Test-Path)) +{ + Write-Error "InputPath should be the path to the root of 'SiteExtension' folder (where 'extension.xml' is)." + Write-Error "Make sure to publish the site extension before running this script." + exit 1 +} + +function New-TemporaryDirectory { + $tmp = [System.IO.Path]::GetTempPath() + $name = (New-Guid).ToString("N") + return New-Item -ItemType Directory -Path (Join-Path $tmp $name) +} + +function Write-Zip ($outputPath) +{ + if (Test-Path $outputPath) { + if ($Force) { + Remove-Item -Path $outputPath -Recurse -Force + } else { + Write-Error "OutputPath already exists. Use -Force to overwrite." + exit 1 + } + } + + $tempDir = New-TemporaryDirectory + Write-Folder $tempDir + + Compress-Archive -Path "$tempDir/*" -DestinationPath $outputPath + Remove-Item -Path $tempDir -Recurse -Force +} + +function Write-Folder ($outputPath) +{ + if (Test-Path "$outputPath/*") { + if ($Force) { + Remove-Item -Path $outputPath -Recurse -Force + } else { + Write-Error "OutputPath already exists. Use -Force to overwrite." + exit 1 + } + } + + $inputPath = Get-ChildItem -Path .\SiteExtension\ -Attributes Directory + $outputPath = Join-Path $outputPath "SiteExtensions" "Functions" + New-Item -ItemType Directory -Path $outputPath | Out-Null + + Copy-Item "$inputPath/applicationHost.xdt" -Destination $outputPath | Out-Null + + $filesDest = (Join-Path $outputPath "32bit") + Copy-Item "$inputPath/32bit/" -Destination "$filesDest/" -Container -Recurse | Out-Null + + $workerDest = (Join-Path $filesDest "workers") + Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null +} + +if ($Zip) { + if (-not $OutputPath.EndsWith(".zip")) { + $OutputPath = "$OutputPath.zip" + } + + Write-Zip $OutputPath +} else { + Write-Folder $OutputPath +} + +Write-Host "Published private site extension to $OutputPath" diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index ea7b2fe591..6235be4414 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -29,8 +29,11 @@ $([MSBuild]::NormalizePath($([System.IO.Path]::Combine($(PublishDir), $(SiteExtensionRelativeDir))))) + + + + - diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index 02c32a7fe3..72a0d0d73b 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -22,4 +22,8 @@ + + + + From d80c7923acac1e0caabd2be52370e89a9579e1c0 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 12 Dec 2024 11:03:50 -0800 Subject: [PATCH 06/15] Copy release_notes.md in site extension --- eng/ci/templates/official/jobs/build-artifacts-windows.yml | 7 ------- .../WebJobs.Script.SiteExtension.csproj | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 471ecb40ea..6747a05e4e 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -96,13 +96,6 @@ jobs: pattern: '*.nupkg' signType: nuget - - task: CopyFiles@2 - displayName: Stage Release Notes - inputs: - SourceFolder: $(Build.SourcesDirectory) - Contents: release_notes.md - TargetFolder: $(windows_drop_path) - - task: DeleteFiles@1 displayName: Delete CodeSignSummary files inputs: diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index 72a0d0d73b..408d5bd82b 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -23,6 +23,7 @@ + From cee5066d1691b2cdbf9b28eec1e586d14262c273 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 12 Dec 2024 15:45:40 -0800 Subject: [PATCH 07/15] Flatten SiteExtension artifacts, custom hash task --- .../New-PrivateSiteExtension.ps1 | 49 +++++++++++++++---- .../Publish.SingleTFM.targets | 20 ++++---- .../Tasks.targets | 42 ++++++++++++++++ 3 files changed, 93 insertions(+), 18 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 index 2f33a85759..3cc1f79894 100644 --- a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -6,10 +6,13 @@ Takes in a published site extension and produces a private site extension. .PARAMETER InputPath - The path of the published 'SiteExtension'. Default is "./SiteExtension". + The path of the published 'SiteExtension'. Leave null to scan for root in a child from here. .PARAMETER OutputPath - The path to produce the private site extension to. When zipping, this is the name of the zip file. Default is "./PrivateSiteExtension". + The path to produce the private site extension to. When zipping, this is the name of the zip file. Leave null to compute this name. + + .PARAMETER Bitness + The bitness to produce the private site extension with. Default is '32bit'. .PARAMETER Zip [Switch] Include to produce site extension as a zip. @@ -25,19 +28,47 @@ #> param ( - [string] $InputPath = "./SiteExtension", - [string] $OutputPath = "./PrivateSiteExtension", + [string] $InputPath = $null, + [string] $OutputPath = $null, + [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '32bit', [switch] $Zip, [switch] $Force ) +$normalizeBitness = @{ + 'x64' = '64bit' + '64bit' = '64bit' + 'x86' = '64bit' + '32bit' = '32bit' +} + +$Bitness = $normalizeBitness[$Bitness] + +if (-not $InputPath) +{ + $InputPath = (Get-ChildItem -Path . -Filter "extension.xml" -Recurse).Directory.FullName +} + +if (Test-Path (Join-Path $InputPath "WebJobs.Script.SiteExtension.csproj")) +{ + Write-Error "This script should not be ran in the WebJobs.Script.SiteExtension project folder. Run this script in the root of the published site extension folder." + exit 1 +} + if (-not (Join-Path $InputPath "extension.xml" | Test-Path)) { - Write-Error "InputPath should be the path to the root of 'SiteExtension' folder (where 'extension.xml' is)." - Write-Error "Make sure to publish the site extension before running this script." + Write-Error "Unable to find published site extension." exit 1 } +if (-not $OutputPath) +{ + $runtime = $Bitness -eq '32bit' ? 'win-x86' : 'win-x64' + $leaf = (Split-Path $InputPath -Leaf) + $split = $leaf.IndexOf('.') + $OutputPath = "$($leaf.Substring(0, $split)).Private.$($leaf.Substring($split + 1)).$runtime" +} + function New-TemporaryDirectory { $tmp = [System.IO.Path]::GetTempPath() $name = (New-Guid).ToString("N") @@ -73,14 +104,14 @@ function Write-Folder ($outputPath) } } - $inputPath = Get-ChildItem -Path .\SiteExtension\ -Attributes Directory + $inputPath = Get-ChildItem -Path $InputPath -Attributes Directory $outputPath = Join-Path $outputPath "SiteExtensions" "Functions" New-Item -ItemType Directory -Path $outputPath | Out-Null Copy-Item "$inputPath/applicationHost.xdt" -Destination $outputPath | Out-Null - $filesDest = (Join-Path $outputPath "32bit") - Copy-Item "$inputPath/32bit/" -Destination "$filesDest/" -Container -Recurse | Out-Null + $filesDest = (Join-Path $outputPath "$Bitness") + Copy-Item "$inputPath/$Bitness/" -Destination "$filesDest/" -Container -Recurse | Out-Null $workerDest = (Join-Path $filesDest "workers") Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index 6235be4414..c56d49d46b 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -12,6 +12,7 @@ PublishProjectReferences; RemoveUnneededRuntimes; MoveSymbols; + ComputeHardLinkHashes; WriteHardLinkHashes; ZipAfterPublish;ZipArtifactsPath @@ -25,7 +26,8 @@ - SiteExtension/$(Version)/ + $(SiteExtensionName).$(Version)/ + $(SiteExtensionBaseDir)$(Version)/ $([MSBuild]::NormalizePath($([System.IO.Path]::Combine($(PublishDir), $(SiteExtensionRelativeDir))))) @@ -34,7 +36,7 @@ - + @@ -72,7 +74,7 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false - + <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> - $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(Identity) $(SiteExtensionName).Symbols.$(Version).%(Identity).zip - <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> - + <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + @@ -118,9 +120,9 @@ <_FilesToHash Include="$(SiteExtensionDir)**" /> - + - + @@ -130,7 +132,7 @@ diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets index 67f3ef84c7..0cca5b0c6c 100644 --- a/src/WebJobs.Script.SiteExtension/Tasks.targets +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -97,4 +97,46 @@ for (int i = 0; i < Directories.Length; i++) + + + + + + + + + + + + + + { + try + { + var file = Files[i]; + using (var stream = File.OpenRead(file.ItemSpec)) + using (var algorithm = MD5.Create()) + { + byte[] hash = algorithm.ComputeHash(stream); + file.SetMetadata("FileHash", Convert.ToBase64String(hash)); + } + } + catch (Exception ex) + { + Log.LogErrorFromException(ex); + } + }); + + Items = Files; + + if (Log.HasLoggedErrors) + { + return false; + } +]]> + + + + From 3e73deceba886b20e2599674dfb679d1efb24b0d Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 12 Dec 2024 16:33:35 -0800 Subject: [PATCH 08/15] Generate release.json during build --- .../New-ReleaseJson.ps1 | 41 +++++++++++++++++++ .../Publish.SingleTFM.targets | 14 ++++++- .../WebJobs.Script.SiteExtension.csproj | 2 +- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 diff --git a/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 b/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 new file mode 100644 index 0000000000..c5c30f9a18 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 @@ -0,0 +1,41 @@ +param( + [string] $Version = $null, + [string[]] $Artifacts = $null, + [string] $CommitId = $null, + [string] $OutputPath +) + +if (-not $Version) { + $Version = dotnet build $PSScriptRoot --getProperty:Version +} + +if (-Not $Artifacts) { + $Artifacts = @( + "Functions.$Version" + "Functions.Symbols.$Version.win-x64" + "Functions.Symbols.$Version.win-x86" + ) +} + +if (-not $CommitId) { + $CommitId = $env:BUILD_SOURCEVERSION +} + +if (-not $CommitId) { + $CommitId = (git rev-parse HEAD).Trim() +} + +$obj = @{ + name = $Version + artifacts = $Artifacts + tag = "v$Version" + commitId = $CommitId + releaseNotesFile = "release_notes.md" +} + +if ($OutputPath) { + Write-Host "Writing $obj to $OutputPath" + $obj | ConvertTo-Json | Out-File $OutputPath +} + +return $obj diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index c56d49d46b..010ba608ed 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -12,8 +12,8 @@ PublishProjectReferences; RemoveUnneededRuntimes; MoveSymbols; - ComputeHardLinkHashes; WriteHardLinkHashes; + WriteReleaseJson; ZipAfterPublish;ZipArtifactsPath @@ -74,6 +74,7 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false + <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> $(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(SiteExtensionName).Symbols.$(Version).%(Identity) $(SiteExtensionName).Symbols.$(Version).%(Identity).zip <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + @@ -136,4 +139,13 @@ Lines="@(_HashedFiles->'Hash: %(FileHash) FileName: %(RelativePath)')" /> + + + <_ReleaseArtifactsArg>@(ReleaseArtifact->'"%(Identity)"', ', ') + + + + + + diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index 408d5bd82b..85e55247ec 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -24,7 +24,7 @@ - + From 3989192d0673c4eb6eeb1a5ac3a0b4c2a53295b4 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 18 Dec 2024 12:25:32 -0800 Subject: [PATCH 09/15] modifyOutputPath: false --- eng/ci/templates/official/jobs/build-artifacts-linux.yml | 1 + eng/ci/templates/steps/build-site-ext.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index 283cd58a2d..1e321d910a 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -58,5 +58,6 @@ jobs: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic + modifyOutputPath: false projects: $(project) arguments: '$(build_args) --no-build -bl:$(log_path)/publish.binlog -o $(linux_drop_path)/host' diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index eff8d581bc..53bcfaacc7 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -31,6 +31,7 @@ steps: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic + modifyOutputPath: false projects: ${{ parameters.project }} ${{ if eq(parameters.publishDir, '') }}: arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog' From 9adf37ff9a0fcaf91c3338b6d10a52b242be68ac Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 18 Dec 2024 13:25:24 -0800 Subject: [PATCH 10/15] Use SHA256 for the hash --- src/WebJobs.Script.SiteExtension/Tasks.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets index 0cca5b0c6c..d3c43a65e8 100644 --- a/src/WebJobs.Script.SiteExtension/Tasks.targets +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -97,7 +97,7 @@ for (int i = 0; i < Directories.Length; i++) - + @@ -116,7 +116,7 @@ for (int i = 0; i < Directories.Length; i++) { var file = Files[i]; using (var stream = File.OpenRead(file.ItemSpec)) - using (var algorithm = MD5.Create()) + using (var algorithm = SHA256.Create()) { byte[] hash = algorithm.ComputeHash(stream); file.SetMetadata("FileHash", Convert.ToBase64String(hash)); From bf2482f07b8ab8070adfc61d39724ff8fa693b30 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 24 Jan 2025 11:54:41 -0800 Subject: [PATCH 11/15] Default private siteext to 64bit --- src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 index 3cc1f79894..6612e5fc2c 100644 --- a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -12,7 +12,7 @@ The path to produce the private site extension to. When zipping, this is the name of the zip file. Leave null to compute this name. .PARAMETER Bitness - The bitness to produce the private site extension with. Default is '32bit'. + The bitness to produce the private site extension with. Default is '64bit'. .PARAMETER Zip [Switch] Include to produce site extension as a zip. @@ -30,7 +30,7 @@ param ( [string] $InputPath = $null, [string] $OutputPath = $null, - [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '32bit', + [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '54bit', [switch] $Zip, [switch] $Force ) From 2d47b2204c323710032e411765c55428ccd42e2f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 24 Jan 2025 11:57:18 -0800 Subject: [PATCH 12/15] Default private siteext to zipping --- .../New-PrivateSiteExtension.ps1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 index 6612e5fc2c..ae7030cd20 100644 --- a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -9,13 +9,13 @@ The path of the published 'SiteExtension'. Leave null to scan for root in a child from here. .PARAMETER OutputPath - The path to produce the private site extension to. When zipping, this is the name of the zip file. Leave null to compute this name. + The path to produce the private site extension to (either the zip file or folder). Leave null to compute this name. .PARAMETER Bitness The bitness to produce the private site extension with. Default is '64bit'. - .PARAMETER Zip - [Switch] Include to produce site extension as a zip. + .PARAMETER NoZip + [Switch] Include to produce site extension as a folder and not a zip. .PARAMETER Force [Switch] Include to overwrite existing files. @@ -31,7 +31,7 @@ param ( [string] $InputPath = $null, [string] $OutputPath = $null, [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '54bit', - [switch] $Zip, + [switch] $NoZip, [switch] $Force ) @@ -117,14 +117,14 @@ function Write-Folder ($outputPath) Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null } -if ($Zip) { +if ($NoZip) { + Write-Folder $OutputPath +} else { if (-not $OutputPath.EndsWith(".zip")) { $OutputPath = "$OutputPath.zip" } Write-Zip $OutputPath -} else { - Write-Folder $OutputPath } Write-Host "Published private site extension to $OutputPath" From 633cc964f07f8fb858caca7870c9d91bbbb6ff8a Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 24 Jan 2025 12:56:26 -0800 Subject: [PATCH 13/15] Remove binlog artifact to save space --- .../official/jobs/build-artifacts-linux.yml | 13 +++---------- .../official/jobs/build-artifacts-windows.yml | 7 ------- eng/ci/templates/steps/build-site-ext.yml | 8 ++++---- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index 1e321d910a..99b053a906 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -7,7 +7,6 @@ jobs: configuration: release runtime: linux-x64 drop_path: $(Build.ArtifactStagingDirectory) - log_path: $(drop_path)/logs linux_drop_path: $(drop_path)/linux build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true' @@ -20,12 +19,6 @@ jobs: displayName: Publish linux artifacts path: $(linux_drop_path) artifact: drop_linux - - output: pipelineArtifact - displayName: Publish logs - path: $(log_path) - artifact: logs_linux - sbomEnabled: false - condition: always() pool: name: 1es-pool-azfunc @@ -41,7 +34,7 @@ jobs: command: custom custom: restore projects: $(project) - arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_path)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 + arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 - task: DotNetCoreCLI@2 displayName: Build @@ -49,7 +42,7 @@ jobs: command: custom custom: build projects: $(project) - arguments: '$(build_args) --no-restore -bl:$(log_path)/build.binlog' + arguments: '$(build_args) --no-restore' - task: DotNetCoreCLI@2 displayName: Publish @@ -60,4 +53,4 @@ jobs: zipAfterPublish: false # we use our own zip logic modifyOutputPath: false projects: $(project) - arguments: '$(build_args) --no-build -bl:$(log_path)/publish.binlog -o $(linux_drop_path)/host' + arguments: '$(build_args) --no-build -o $(linux_drop_path)/host' diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 36ce685723..0d4daf75d0 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -11,7 +11,6 @@ jobs: drop_path: $(Build.ArtifactStagingDirectory) nuget_drop_path: $(drop_path)/nuget windows_drop_path: $(drop_path)/windows - log_path: $(drop_path)/log templateContext: outputParentDirectory: $(drop_path) @@ -24,12 +23,6 @@ jobs: displayName: Publish nuget packages path: $(nuget_drop_path) artifact: drop_nuget - - output: pipelineArtifact - displayName: Publish logs - path: $(log_path) - artifact: logs_windows - sbomEnabled: false - condition: always() steps: - template: /eng/ci/templates/install-dotnet.yml@self diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index 53bcfaacc7..7b59a61a8c 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -14,7 +14,7 @@ steps: command: custom custom: restore projects: ${{ parameters.project }} - arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_path)/site_ext.restore.binlog' + arguments: '-v m -p:PublishReadyToRun=true' - task: DotNetCoreCLI@2 displayName: Build site extension @@ -22,7 +22,7 @@ steps: command: custom custom: build projects: ${{ parameters.project }} - arguments: '--no-restore -v m -c release -bl:$(log_path)/site_ext.build.binlog' + arguments: '--no-restore -v m -c release' - task: DotNetCoreCLI@2 displayName: Publish site extension @@ -34,6 +34,6 @@ steps: modifyOutputPath: false projects: ${{ parameters.project }} ${{ if eq(parameters.publishDir, '') }}: - arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog' + arguments: '--no-build -v m -c release' ${{ else }}: - arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog -o ${{ parameters.publishDir }}' + arguments: '--no-build -v m -c release -o ${{ parameters.publishDir }}' From cffdc72f095eb8a8eb850657cb6087fc59e7772f Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 24 Jan 2025 13:02:25 -0800 Subject: [PATCH 14/15] Update SiteExtension readme --- src/WebJobs.Script.SiteExtension/readme.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/WebJobs.Script.SiteExtension/readme.md b/src/WebJobs.Script.SiteExtension/readme.md index 1e01261a06..0d480c1809 100644 --- a/src/WebJobs.Script.SiteExtension/readme.md +++ b/src/WebJobs.Script.SiteExtension/readme.md @@ -22,3 +22,20 @@ By default the outputs will not be zipped. To the zip the final outputs, add `-p ## Outputs The output site extension can be found at `{repo_root}/out/pub/WebJobs.Script.SiteExtension/{config}_win`. When using `-p:ZipAfterPublish=true`, the zipped package is found at `{repo_root}/out/pkg/{config}` + +## Private Site Extension + +Private site extension (PSE) is not generated as part of building this project. To get a private site extension, navigate to the [publish output](#outputs) and run `New-PrivateSiteExtension.ps1` + + +``` powershell +# Generates a zipped x64 PSE by default +./New-PrivateSiteExtension.ps1 + +# To generate x86 / 32bit: +./New-PrivateSiteExtension.ps1 -Bitness x86 + + +# Can skip zipping the extension: +./New-PrivateSiteExtension.ps1 -NoZip +``` From 415eb87b1db3f78451c7cd49a5cde5db7cf9a002 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Fri, 24 Jan 2025 13:29:39 -0800 Subject: [PATCH 15/15] Update src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 Co-authored-by: Lilian Kasem --- src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 index ae7030cd20..9f1ab167a8 100644 --- a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -30,7 +30,7 @@ param ( [string] $InputPath = $null, [string] $OutputPath = $null, - [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '54bit', + [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '64bit', [switch] $NoZip, [switch] $Force )