Skip to content

Commit

Permalink
Use Release Studio for public publish (#1270) (#1271)
Browse files Browse the repository at this point in the history
Includes a more detailed assets.json so the new,
more unique URLs are discoverable.

(cherry picked from commit 4997d4d)
  • Loading branch information
dagood authored Jul 19, 2024
1 parent 60d7064 commit f36ffa4
Show file tree
Hide file tree
Showing 14 changed files with 447 additions and 411 deletions.
4 changes: 2 additions & 2 deletions eng/_util/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ module github.com/microsoft/go/_util
go 1.16

require (
github.com/microsoft/go-infra v0.0.0-20231219225928-f2e20f366e2d
github.com/microsoft/go-infra v0.0.3
github.com/microsoft/go/_core v0.0.0
golang.org/x/sys v0.13.0
golang.org/x/sys v0.22.0
gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d
)

Expand Down
409 changes: 52 additions & 357 deletions eng/_util/go.sum

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions eng/pipeline/rolling-internal-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ parameters:
type: boolean
default: false

- name: publishReleaseStudio
displayName: >
[Release automation input]
Publish artifacts to the public using Release Studio integration.
To save resources, leave false for builds that definitely won't be released, like microsoft/main rolling builds.
type: boolean
default: false

- name: publishExistingRunID
displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.'
type: string
Expand Down Expand Up @@ -67,5 +75,6 @@ extends:
createSourceArchive: true
createSymbols: true
publish: true
publishReleaseStudio: ${{ parameters.publishReleaseStudio }}
publishExistingRunID: ${{ parameters.publishExistingRunID }}
releaseVersion: ${{ parameters.releaseVersion }}
16 changes: 6 additions & 10 deletions eng/pipeline/stages/builders-to-stages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ parameters:
builders: []
# If true, publish build artifacts to blob storage.
publish: false
# If true, publish artifacts to the public using Release Studio integration.
publishReleaseStudio: false
# If changed to specify an existing pipeline run, skip build/sign and publish the existing run.
publishExistingRunID: 'nil'
# If true, include a signing stage+job that depends on all 'buildandpack' builder jobs finishing.
Expand Down Expand Up @@ -54,18 +56,12 @@ stages:
- ${{ builder }}

- ${{ if eq(parameters.publish, true) }}:
- ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
- template: pool.yml
- ${{ if and(not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')), eq(parameters.publishReleaseStudio, true)) }}:
- template: public-publish-stages.yml
parameters:
inner:
template: publish-stage.yml
parameters:
# This is not a builder, but provide partial builder info for agent selection.
builder: { os: windows, arch: amd64 }
official: true
public: true
publishExistingRunID: ${{ parameters.publishExistingRunID }}
publishExistingRunID: ${{ parameters.publishExistingRunID }}

# Always publish the results to a private storage account and publish symbols to AzDO.
- template: pool.yml
parameters:
inner:
Expand Down
4 changes: 4 additions & 0 deletions eng/pipeline/stages/go-builder-matrix-stages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ parameters:
- name: releaseVersion
type: string
default: 'nil'
- name: publishReleaseStudio
type: boolean
default: false
- name: publishExistingRunID
type: string
default: 'nil'
Expand All @@ -51,6 +54,7 @@ stages:
jobsParameters:
official: ${{ parameters.official }}
publish: ${{ parameters.publish }}
publishReleaseStudio: ${{ parameters.publishReleaseStudio }}
publishExistingRunID: ${{ parameters.publishExistingRunID }}
createSourceArchive: ${{ parameters.createSourceArchive }}
createSymbols: ${{ parameters.createSymbols }}
Expand Down
77 changes: 77 additions & 0 deletions eng/pipeline/stages/public-publish-stage.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Copyright (c) Microsoft Corporation.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# This stage performs common setup for each of the several stages involved in
# public publishing using Release Studio. It gets the utilities needed to
# publish the build artifacts publicly, executes common logic needed by each
# stage involved, and then executes some provided steps.

parameters:
- name: id
type: string

- name: dependsOn
type: object
default: []

- name: templateContext
type: object
default: {}

- name: variables
type: object
default: []

- name: steps
type: object

stages:
- stage: ${{ parameters.id }}
dependsOn: ${{ parameters.dependsOn }}

jobs:
- job: ${{ parameters.id }}
pool:
name: DotNet-Publishing-1ES-MI
workspace:
clean: all
timeoutInMinutes: 240 # 4hr
${{ if parameters.templateContext }}:
templateContext: ${{ parameters.templateContext }}
variables:
# Group includes these relevant variables:
# - MI_ClientId
# - ReleaseStudioPublishOwnerAliases
- group: go-release-config

- name: GoBinariesDir
value: $(Pipeline.Workspace)/Binaries Signed
- name: NetCorePublishingScripts
value: $(Build.SourcesDirectory)/eng/release-studio/bin/Debug/net7.0/MsGoPublishingScripts

# This is read directly by release studio scripts through env.
- name: RMExecutionEnvironment
value: Production

- ${{ each variable in parameters.variables }}:
- ${{ variable }}
steps:
- template: ../steps/checkout-windows-task.yml
- template: ../steps/find-PublishBranchAlias-task.yml

- task: NuGetAuthenticate@1
displayName: 'Authenticate to AzDO Feeds'

- task: DotNetCoreCLI@2
displayName: 'Acquire tools using csproj'
inputs:
command: custom
projects: '$(Build.SourcesDirectory)/eng/release-studio/ReleaseStudio.csproj'
custom: build
arguments: >-
/bl:eng/release-studio/Build.binlog
/v:n
- ${{ each step in parameters.steps }}:
- ${{ step }}
189 changes: 189 additions & 0 deletions eng/pipeline/stages/public-publish-stages.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
# Copyright (c) Microsoft Corporation.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# These stages publish a build's artifacts to the public location using Release
# Studio integration.

parameters:
- name: publishExistingRunID
type: string
default: 'nil'

stages:
- template: public-publish-stage.yml
parameters:
id: PublishPublicBinaries

${{ if eq(parameters.publishExistingRunID, 'nil') }}:
dependsOn: Sign
${{ else }}:
dependsOn: []

steps:
- template: ../steps/download-signed-binaries-task.yml
parameters:
runID: ${{ parameters.publishExistingRunID }}

- task: PowerShell@2
displayName: ➡️ Send Publishing Request To Release Studio
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.SendPublishRequest.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-PublishingFilesPath "$(GoBinariesDir)"
-BuildNumber "$(Build.BuildNumber)"
-BranchName "$(PublishBranchAlias)"
-OwnerAliases $(ReleaseStudioPublishOwnerAliases)
-ReleaseIdVariableName 'MsGoPublishRequestIdBinaries'

# The publish request script sets this variable, but doesn't mark it as an output.
- pwsh: echo '##vso[task.setvariable variable=MsGoPublishRequestIdBinaries;isOutput=true]$(MsGoPublishRequestIdBinaries)'
displayName: 'Set MsGoPublishRequestIdBinaries isOutput=true'
name: OutVariables

- template: public-publish-stage.yml
parameters:
id: WaitPublicBinaries
dependsOn: PublishPublicBinaries
variables:
- name: RequestID
value: $[ stageDependencies.PublishPublicBinaries.PublishPublicBinaries.outputs['OutVariables.MsGoPublishRequestIdBinaries'] ]
- name: ReleaseStudioBinariesManifestDir
value: '$(Pipeline.Workspace)/ReleaseStudioBinariesManifest'
templateContext:
outputs:
- output: pipelineArtifact
path: $(ReleaseStudioBinariesManifestDir)
artifact: Release Studio Binaries Manifest
steps:
- task: PowerShell@2
displayName: ⌚ Wait for Publishing Request Completion
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.WaitForPublishingRequestToBeLive.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-RequestId "$(RequestID)"

- pwsh: New-Item "$(ReleaseStudioBinariesManifestDir)" -ItemType Directory -ErrorAction Ignore
displayName: 'Create Publish Manifest File Output Directory'

- task: PowerShell@2
displayName: 🗃️ Create Publish Manifest File
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.CreateOutputManifestFile.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-OutputDirectory "$(ReleaseStudioBinariesManifestDir)"
-RequestId "$(RequestID)"

- template: public-publish-stage.yml
parameters:
id: PublishAssetManifest
dependsOn: WaitPublicBinaries
variables:
- name: BuildAssetsPrePublishDir
value: '$(Pipeline.Workspace)/BuildAssetsPrePublishDir'
templateContext:
outputs:
- output: pipelineArtifact
path: $(BuildAssetsPrePublishDir)
artifact: BuildAssetsPrePublish
steps:
- template: ../steps/download-signed-binaries-task.yml
parameters:
runID: ${{ parameters.publishExistingRunID }}

- download: current
artifact: Release Studio Binaries Manifest
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
displayName: 'Download: Release Studio Binaries Manifest'

- pwsh: |
New-Item "$(BuildAssetsPrePublishDir)" -ItemType Directory -ErrorAction Ignore
eng/run.ps1 createbuildassetjson `
-artifacts-dir '$(GoBinariesDir)' `
-source-dir '$(Build.SourcesDirectory)' `
-destination-manifest-file '$(Pipeline.Workspace)/Release Studio Binaries Manifest/msGo.output.manifest.json' `
-branch '$(PublishBranchAlias)' `
-o '$(BuildAssetsPrePublishDir)/assets.json'
displayName: 🏗️ Create build asset JSON
- task: PowerShell@2
displayName: ➡️ Send Publishing Request To Release Studio
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.SendPublishRequest.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-PublishingFilesPath "$(BuildAssetsPrePublishDir)"
-BuildNumber "$(Build.BuildNumber)"
-BranchName "$(PublishBranchAlias)"
-OwnerAliases $(ReleaseStudioPublishOwnerAliases)
-ReleaseIdVariableName 'MsGoPublishRequestIdManifest'

# The publish request script sets this variable, but doesn't mark it as an output.
- pwsh: echo '##vso[task.setvariable variable=MsGoPublishRequestIdManifest;isOutput=true]$(MsGoPublishRequestIdManifest)'
displayName: 'Set MsGoPublishRequestIdManifest isOutput=true'
name: OutVariables

- template: public-publish-stage.yml
parameters:
id: WaitAssetManifest
dependsOn: PublishAssetManifest
variables:
- name: RequestID
value: $[ stageDependencies.PublishAssetManifest.PublishAssetManifest.outputs['OutVariables.MsGoPublishRequestIdManifest'] ]
- name: ReleaseStudioMetaManifestDir
value: '$(Pipeline.Workspace)/ReleaseStudioMetaManifest'
- name: PublicBuildAssetsDir
value: '$(Pipeline.Workspace)/PublicBuildAssetsDir'
templateContext:
outputs:
- output: pipelineArtifact
path: $(PublicBuildAssetsDir)
artifact: BuildAssets
- output: pipelineArtifact
path: $(ReleaseStudioMetaManifestDir)
artifact: Release Studio Meta Manifest
steps:
- task: PowerShell@2
displayName: ⌚ Wait for Publishing Request Completion
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.WaitForPublishingRequestToBeLive.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-RequestId "$(RequestID)"

- pwsh: New-Item "$(ReleaseStudioMetaManifestDir)" -ItemType Directory -ErrorAction Ignore
displayName: 'Create Meta Manifest File Output Directory'

- task: PowerShell@2
displayName: 🗃️ Create Meta Manifest File
inputs:
targetType: filePath
filePath: '$(NetCorePublishingScripts)/RM.MsGo.CreateOutputManifestFile.ps1'
arguments:
-MIClientId "$(MI_ClientId)"
-OutputDirectory "$(ReleaseStudioMetaManifestDir)"
-RequestId "$(RequestID)"

- download: current
artifact: BuildAssetsPrePublish
patterns: '!_manifest/**'
displayName: 'Download: BuildAssetsPrePublish'

# Assemble the final BuildAssets artifact. Include the publish manifest in the artifact so
# the release pipeline knows where to find its public URL.
- pwsh: |
New-Item "$(PublicBuildAssetsDir)" -ItemType Directory -ErrorAction Ignore
# Combine outputs. Put Release Studio output in its own directory.
Copy-Item -Path "$(Pipeline.Workspace)\BuildAssetsPrePublish\*" -Destination "$(PublicBuildAssetsDir)"
Copy-Item -Path "$(ReleaseStudioMetaManifestDir)" -Destination "$(PublicBuildAssetsDir)" -Recurse
displayName: 📦 Assemble BuildAssets artifact
Loading

0 comments on commit f36ffa4

Please sign in to comment.