From 1d4838b41ad50644c10e35923f2fa273af5daf66 Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Thu, 5 Sep 2024 08:19:18 -0700 Subject: [PATCH] Update common Docker engineering infrastructure with latest --- eng/common/templates/1es-official.yml | 8 ++-- eng/common/templates/1es-unofficial.yml | 11 ++++- .../templates/jobs/cg-build-projects.yml | 4 +- eng/common/templates/jobs/publish.yml | 39 ++++++++++++++++++ .../stages/dotnet/build-test-publish-repo.yml | 34 +++++++++------- .../templates/steps/annotate-eol-digests.yml | 6 ++- eng/common/templates/variables/common.yml | 12 ++++++ .../templates/variables/docker-images.yml | 2 +- .../templates/variables/dotnet/common.yml | 40 +++++++++++++++++++ 9 files changed, 132 insertions(+), 24 deletions(-) diff --git a/eng/common/templates/1es-official.yml b/eng/common/templates/1es-official.yml index f557156d6..60091b72f 100644 --- a/eng/common/templates/1es-official.yml +++ b/eng/common/templates/1es-official.yml @@ -20,14 +20,14 @@ parameters: - name: pool type: object default: - name: NetCore1ESPool-Internal - image: 1es-ubuntu-2204 + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) os: linux - name: sourceAnalysisPool type: object default: - name: NetCore1ESPool-Internal - image: 1es-windows-2022 + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) os: windows resources: diff --git a/eng/common/templates/1es-unofficial.yml b/eng/common/templates/1es-unofficial.yml index b1bc12a49..30faab042 100644 --- a/eng/common/templates/1es-unofficial.yml +++ b/eng/common/templates/1es-unofficial.yml @@ -23,8 +23,14 @@ parameters: - name: pool type: object default: - name: NetCore1ESPool-Internal - image: 1es-windows-2022 + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) os: windows resources: @@ -50,4 +56,5 @@ extends: exclude: - repository: InternalVersionsRepo - repository: PublicVersionsRepo + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} stages: ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml index 976a06783..9b56c288d 100644 --- a/eng/common/templates/jobs/cg-build-projects.yml +++ b/eng/common/templates/jobs/cg-build-projects.yml @@ -12,8 +12,8 @@ jobs: - job: BuildProjects displayName: Build Projects pool: - name: NetCore1ESPool-Internal - image: 1es-ubuntu-2204 + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) os: linux steps: - powershell: > diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index a2bb7f3c1..9b8e86178 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -7,6 +7,7 @@ parameters: jobs: - job: Publish pool: ${{ parameters.pool }} + timeoutInMinutes: 90 variables: - name: imageBuilder.commonCmdArgs value: > @@ -27,6 +28,8 @@ jobs: value: $(artifactsPath)/imageInfo - name: sourceBuildIdOutputDir value: $(Build.ArtifactStagingDirectory)/sourceBuildId + - name: imageInfoRawContentUrl + value: https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath) - ${{ parameters.customPublishVariables }} steps: - template: /eng/common/templates/steps/retain-build.yml@self @@ -103,6 +106,10 @@ jobs: parameters: dryRunArg: $(dryRunArg) condition: and(succeeded(), eq(variables['publishReadme'], 'true')) + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data + curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-old.json $(imageInfoRawContentUrl) + displayName: Download Original Image Info File - script: > $(runImageBuilderCmd) publishImageInfo '$(imageInfoContainerDir)/image-info.json' @@ -117,6 +124,8 @@ jobs: $(imageBuilder.commonCmdArgs) condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) displayName: Publish Image Info + - script: curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-new.json $(imageInfoRawContentUrl) + displayName: Download Updated Image Info File - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Ingest Kusto Image Info @@ -134,6 +143,33 @@ jobs: --architecture '*' $(dryRunArg) $(imageBuilder.commonCmdArgs) + - template: /eng/common/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Generate EOL Annotation Data + serviceConnection: $(publish.serviceConnectionName) + internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + args: > + generateEolAnnotationData + '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' + '$(artifactsPath)/eol-annotation-data/image-info-old.json' + '$(artifactsPath)/eol-annotation-data/image-info-new.json' + '$(publishRepoPrefix)' + '$(acr.server)' + $(generateEolAnnotationDataExtraOptions) + $(dryRunArg) + - template: /eng/common/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/eol-annotation-data + artifactName: eol-annotation-data-$(System.JobAttempt) + displayName: Publish EOL Annotation Data Artifact + internalProjectName: internal + publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) + - template: /eng/common/templates/steps/annotate-eol-digests.yml@self + parameters: + internalProjectName: ${{ parameters.internalProjectName }} + dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - script: > $(runImageBuilderCmd) postPublishNotification '$(publishNotificationRepoName)' @@ -154,6 +190,9 @@ jobs: --task "Wait for MCR Doc Ingestion (Authenticated)" --task "Publish Image Info" --task "Ingest Kusto Image Info (Authenticated)" + --task "Generate EOL Annotation Data (Authenticated)" + --task "Annotate EOL Images (Authenticated)" + --task "Wait for Annotation Ingestion (Authenticated)" $(dryRunArg) $(imageBuilder.commonCmdArgs) displayName: Post Publish Notification diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index 71b044c96..53f9e8a63 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -70,47 +70,53 @@ stages: ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}: vmImage: $(defaultLinuxAmd64PoolImage) ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: NetCore1ESPool-Internal - image: 1es-ubuntu-2204 + name: $(linuxAmd64InternalPoolName) + image: $(linuxAmd64InternalPoolImage) os: linux # Linux Arm64 linuxArm64Pool: os: linux hostArchitecture: Arm64 - image: Mariner-2-Docker-ARM64 + image: $(linuxArm64PoolImage) ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - name: Docker-Linux-Arm-Public + name: $(linuxArm64PublicPoolName) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: Docker-Linux-Arm-Internal + name: $(linuxArm64InternalPoolName) # Linux Arm32 linuxArm32Pool: os: linux hostArchitecture: Arm64 - image: Mariner-2-Docker-ARM64 + image: $(linuxArm32PoolImage) ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: - name: Docker-Linux-Arm-Public + name: $(linuxArm32PublicPoolName) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - name: Docker-Linux-Arm-Internal + name: $(linuxArm32InternalPoolName) # Windows Server 2016 windows2016Pool: os: windows - name: Docker-2016-${{ variables['System.TeamProject'] }} + name: $(windowsServer2016PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2016PublicPoolImage) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: Server2016-NESDockerBuilds-PT + image: $(windowsServer2016InternalPoolImage) # Windows Server 2019 (1809) windows1809Pool: os: windows - name: Docker-1809-${{ variables['System.TeamProject'] }} + name: $(windowsServer2019PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2019PublicPoolImage) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: Server2019-1809-NESDockerBuilds-1ESPT + image: $(windowsServer2019InternalPoolImage) # Windows Server 2022 windows2022Pool: os: windows - name: Docker-2022-${{ variables['System.TeamProject'] }} + name: $(windowsServer2022PoolName) + ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}: + image: $(windowsServer2022PublicPoolImage) ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - image: Server2022-NESDockerBuilds-1ESPT + image: $(windowsServer2022InternalPoolImage) diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml index 92c9199eb..3f01cad02 100644 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ b/eng/common/templates/steps/annotate-eol-digests.yml @@ -9,12 +9,14 @@ steps: displayName: Annotate EOL Images serviceConnection: $(publish.serviceConnectionName) internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: > annotateEolDigests - /repo/${{ parameters.dataFile }} + ${{ parameters.dataFile }} $(acr.server) $(publishRepoPrefix) $(artifactsPath)/annotation-digests/annotation-digests.txt + $(dryRunArg) - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(Build.ArtifactStagingDirectory)/annotation-digests @@ -22,11 +24,13 @@ steps: displayName: Publish Annotation Digests List internalProjectName: internal publicProjectName: public + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Wait for Annotation Ingestion serviceConnection: $(marStatus.serviceConnectionName) internalProjectName: internal + condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: > waitForMarAnnotationIngestion $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/common/templates/variables/common.yml b/eng/common/templates/variables/common.yml index 4158ffbba..7b0aafe45 100644 --- a/eng/common/templates/variables/common.yml +++ b/eng/common/templates/variables/common.yml @@ -17,6 +17,8 @@ variables: value: "" - name: imageBuilderDockerRunExtraOptions value: "" +- name: generateEolAnnotationDataExtraOptions + value: "" - name: productVersionComponents value: 2 - name: imageInfoVariant @@ -56,6 +58,16 @@ variables: - name: defaultWindows2022PoolImage value: windows-2022 +- name: default1ESInternalPoolName + value: NetCore1ESPool-Internal +- name: default1ESInternalPoolImage + value: 1es-ubuntu-2204 + +- name: defaultSourceAnalysisPoolName + value: NetCore1ESPool-Internal +- name: defaultSourceAnalysisPoolImage + value: 1es-windows-2022 + # Define these as placeholder values to allow string validation to succeed since we don't have the # variable group with the actual values in public builds. For internal builds, the variable group # will cause these values to be overridden with the real values. diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 028c5f193..cabe3f2f3 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2516503 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2530171 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner diff --git a/eng/common/templates/variables/dotnet/common.yml b/eng/common/templates/variables/dotnet/common.yml index 64b77430c..70bbdc003 100644 --- a/eng/common/templates/variables/dotnet/common.yml +++ b/eng/common/templates/variables/dotnet/common.yml @@ -7,6 +7,46 @@ variables: - name: dockerHubRegistryCreds value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' +- name: linuxAmd64InternalPoolImage + value: 1es-ubuntu-2204 +- name: linuxAmd64InternalPoolName + value: NetCore1ESPool-Internal + +- name: linuxArm64PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm64PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm64InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: linuxArm32PoolImage + value: Mariner-2-Docker-ARM64 +- name: linuxArm32PublicPoolName + value: Docker-Linux-Arm-Public +- name: linuxArm32InternalPoolName + value: Docker-Linux-Arm-Internal + +- name: windowsServer2016PublicPoolImage + value: Server2016-NESDockerBuilds +- name: windowsServer2016InternalPoolImage + value: Server2016-NESDockerBuilds-PT +- name: windowsServer2016PoolName + value: Docker-2016-${{ variables['System.TeamProject'] }} + +- name: windowsServer2019PublicPoolImage + value: Server2019-1809-NESDockerBuilds +- name: windowsServer2019InternalPoolImage + value: Server2019-1809-NESDockerBuilds-1ESPT +- name: windowsServer2019PoolName + value: Docker-1809-${{ variables['System.TeamProject'] }} + +- name: windowsServer2022PublicPoolImage + value: Server2022-NESDockerBuilds +- name: windowsServer2022InternalPoolImage + value: Server2022-NESDockerBuilds-1ESPT +- name: windowsServer2022PoolName + value: Docker-2022-${{ variables['System.TeamProject'] }} + - group: DotNet-Docker-Common - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-Docker-Secrets-WIF