[main] Update common Docker engineering infrastructure with latest (#…

…1196)

eng/common/templates/1es-official.yml

@@ -20,14 +20,14 @@ parameters:
 - name: pool
   type: object
   default:
-    name: NetCore1ESPool-Internal
-    image: 1es-ubuntu-2204
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
     os: linux
 - name: sourceAnalysisPool
   type: object
   default:
-    name: NetCore1ESPool-Internal
-    image: 1es-windows-2022
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
     os: windows
 
 resources:

eng/common/templates/1es-unofficial.yml

@@ -23,8 +23,14 @@ parameters:
 - name: pool
   type: object
   default:
-    name: NetCore1ESPool-Internal
-    image: 1es-windows-2022
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
+    os: linux
+- name: sourceAnalysisPool
+  type: object
+  default:
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
     os: windows
 
 resources:
@@ -50,4 +56,5 @@ extends:
         exclude:
         - repository: InternalVersionsRepo
         - repository: PublicVersionsRepo
+      sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
     stages: ${{ parameters.stages }}

eng/common/templates/jobs/cg-build-projects.yml

@@ -12,8 +12,8 @@ jobs:
 - job: BuildProjects
   displayName: Build Projects
   pool:
-    name: NetCore1ESPool-Internal
-    image: 1es-ubuntu-2204
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
     os: linux
   steps:
   - powershell: >

eng/common/templates/jobs/publish.yml

@@ -7,6 +7,7 @@ parameters:
 jobs:
 - job: Publish
   pool: ${{ parameters.pool }}
+  timeoutInMinutes: 90
   variables:
   - name: imageBuilder.commonCmdArgs
     value: >
@@ -27,6 +28,8 @@ jobs:
     value: $(artifactsPath)/imageInfo
   - name: sourceBuildIdOutputDir
     value: $(Build.ArtifactStagingDirectory)/sourceBuildId
+  - name: imageInfoRawContentUrl
+    value: https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)
   - ${{ parameters.customPublishVariables }}
   steps:
   - template: /eng/common/templates/steps/retain-build.yml@self
@@ -103,6 +106,10 @@ jobs:
     parameters:
       dryRunArg: $(dryRunArg)
       condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
+  - script: |
+      mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
+      curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-old.json $(imageInfoRawContentUrl)
+    displayName: Download Original Image Info File
   - script: >
       $(runImageBuilderCmd) publishImageInfo
       '$(imageInfoContainerDir)/image-info.json'
@@ -117,6 +124,8 @@ jobs:
       $(imageBuilder.commonCmdArgs)
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
     displayName: Publish Image Info
+  - script: curl -fSL --output $(Build.ArtifactStagingDirectory)/eol-annotation-data/image-info-new.json $(imageInfoRawContentUrl)
+    displayName: Download Updated Image Info File
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Ingest Kusto Image Info
@@ -134,6 +143,33 @@ jobs:
         --architecture '*'
         $(dryRunArg)
         $(imageBuilder.commonCmdArgs)
+  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: Generate EOL Annotation Data
+      serviceConnection: $(publish.serviceConnectionName)
+      internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
+      args: >
+        generateEolAnnotationData
+        '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
+        '$(artifactsPath)/eol-annotation-data/image-info-old.json'
+        '$(artifactsPath)/eol-annotation-data/image-info-new.json'
+        '$(publishRepoPrefix)'
+        '$(acr.server)'
+        $(generateEolAnnotationDataExtraOptions)
+        $(dryRunArg)
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/eol-annotation-data
+      artifactName: eol-annotation-data-$(System.JobAttempt)
+      displayName: Publish EOL Annotation Data Artifact
+      internalProjectName: internal
+      publicProjectName: public
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
+  - template: /eng/common/templates/steps/annotate-eol-digests.yml@self
+    parameters:
+      internalProjectName: ${{ parameters.internalProjectName }}
+      dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
   - script: >
       $(runImageBuilderCmd) postPublishNotification
       '$(publishNotificationRepoName)'
@@ -154,6 +190,9 @@ jobs:
       --task "Wait for MCR Doc Ingestion (Authenticated)"
       --task "Publish Image Info"
       --task "Ingest Kusto Image Info (Authenticated)"
+      --task "Generate EOL Annotation Data (Authenticated)"
+      --task "Annotate EOL Images (Authenticated)"
+      --task "Wait for Annotation Ingestion (Authenticated)"
       $(dryRunArg)
       $(imageBuilder.commonCmdArgs)
     displayName: Post Publish Notification

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -70,47 +70,53 @@ stages:
       ${{ elseif eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
         vmImage: $(defaultLinuxAmd64PoolImage)
       ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: NetCore1ESPool-Internal
-        image: 1es-ubuntu-2204
+        name: $(linuxAmd64InternalPoolName)
+        image: $(linuxAmd64InternalPoolImage)
         os: linux
 
     # Linux Arm64
     linuxArm64Pool:
       os: linux
       hostArchitecture: Arm64
-      image: Mariner-2-Docker-ARM64
+      image: $(linuxArm64PoolImage)
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
-        name: Docker-Linux-Arm-Public
+        name: $(linuxArm64PublicPoolName)
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: Docker-Linux-Arm-Internal
+        name: $(linuxArm64InternalPoolName)
 
     # Linux Arm32
     linuxArm32Pool:
       os: linux
       hostArchitecture: Arm64
-      image: Mariner-2-Docker-ARM64
+      image: $(linuxArm32PoolImage)
       ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
-        name: Docker-Linux-Arm-Public
+        name: $(linuxArm32PublicPoolName)
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        name: Docker-Linux-Arm-Internal
+        name: $(linuxArm32InternalPoolName)
 
     # Windows Server 2016
     windows2016Pool:
       os: windows
-      name: Docker-2016-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2016PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2016PublicPoolImage)
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2016-NESDockerBuilds-PT
+        image: $(windowsServer2016InternalPoolImage)
 
     # Windows Server 2019 (1809)
     windows1809Pool:
       os: windows
-      name: Docker-1809-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2019PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2019PublicPoolImage)
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2019-1809-NESDockerBuilds-1ESPT
+        image: $(windowsServer2019InternalPoolImage)
 
     # Windows Server 2022
     windows2022Pool:
       os: windows
-      name: Docker-2022-${{ variables['System.TeamProject'] }}
+      name: $(windowsServer2022PoolName)
+      ${{ if eq(variables['System.TeamProject'], parameters.publicProjectName) }}:
+        image: $(windowsServer2022PublicPoolImage)
       ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}:
-        image: Server2022-NESDockerBuilds-1ESPT
+        image: $(windowsServer2022InternalPoolImage)

eng/common/templates/steps/annotate-eol-digests.yml

@@ -9,24 +9,28 @@ steps:
       displayName: Annotate EOL Images
       serviceConnection: $(publish.serviceConnectionName)
       internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
       args: >
         annotateEolDigests
-        /repo/${{ parameters.dataFile }}
+        ${{ parameters.dataFile }}
         $(acr.server)
         $(publishRepoPrefix)
         $(artifactsPath)/annotation-digests/annotation-digests.txt
+        $(dryRunArg)
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
       path: $(Build.ArtifactStagingDirectory)/annotation-digests
       artifactName: annotation-digests-$(System.JobAttempt)
       displayName: Publish Annotation Digests List
       internalProjectName: internal
       publicProjectName: public
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Wait for Annotation Ingestion
       serviceConnection: $(marStatus.serviceConnectionName)
       internalProjectName: internal
+      condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
       args: >
         waitForMarAnnotationIngestion
         $(artifactsPath)/annotation-digests/annotation-digests.txt

eng/common/templates/variables/common.yml

@@ -17,6 +17,8 @@ variables:
   value: ""
 - name: imageBuilderDockerRunExtraOptions
   value: ""
+- name: generateEolAnnotationDataExtraOptions
+  value: ""
 - name: productVersionComponents
   value: 2
 - name: imageInfoVariant
@@ -56,6 +58,16 @@ variables:
 - name: defaultWindows2022PoolImage
   value: windows-2022
 
+- name: default1ESInternalPoolName
+  value: NetCore1ESPool-Internal
+- name: default1ESInternalPoolImage
+  value: 1es-ubuntu-2204
+
+- name: defaultSourceAnalysisPoolName
+  value: NetCore1ESPool-Internal
+- name: defaultSourceAnalysisPoolImage
+  value: 1es-windows-2022
+
 # Define these as placeholder values to allow string validation to succeed since we don't have the
 # variable group with the actual values in public builds. For internal builds, the variable group
 # will cause these values to be overridden with the real values.

eng/common/templates/variables/docker-images.yml

@@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2516503
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2530171
   imageNames.imageBuilder: $(imageNames.imageBuilderName)
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
   imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner

eng/common/templates/variables/dotnet/common.yml

@@ -7,6 +7,46 @@ variables:
 - name: dockerHubRegistryCreds
   value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)'
 
+- name: linuxAmd64InternalPoolImage
+  value: 1es-ubuntu-2204
+- name: linuxAmd64InternalPoolName
+  value: NetCore1ESPool-Internal
+
+- name: linuxArm64PoolImage
+  value: Mariner-2-Docker-ARM64
+- name: linuxArm64PublicPoolName
+  value: Docker-Linux-Arm-Public
+- name: linuxArm64InternalPoolName
+  value: Docker-Linux-Arm-Internal
+
+- name: linuxArm32PoolImage
+  value: Mariner-2-Docker-ARM64
+- name: linuxArm32PublicPoolName
+  value: Docker-Linux-Arm-Public
+- name: linuxArm32InternalPoolName
+  value: Docker-Linux-Arm-Internal
+
+- name: windowsServer2016PublicPoolImage
+  value: Server2016-NESDockerBuilds
+- name: windowsServer2016InternalPoolImage
+  value: Server2016-NESDockerBuilds-PT
+- name: windowsServer2016PoolName
+  value: Docker-2016-${{ variables['System.TeamProject'] }}
+
+- name: windowsServer2019PublicPoolImage
+  value: Server2019-1809-NESDockerBuilds
+- name: windowsServer2019InternalPoolImage
+  value: Server2019-1809-NESDockerBuilds-1ESPT
+- name: windowsServer2019PoolName
+  value: Docker-1809-${{ variables['System.TeamProject'] }}
+
+- name: windowsServer2022PublicPoolImage
+  value: Server2022-NESDockerBuilds
+- name: windowsServer2022InternalPoolImage
+  value: Server2022-NESDockerBuilds-1ESPT
+- name: windowsServer2022PoolName
+  value: Docker-2022-${{ variables['System.TeamProject'] }}
+
 - group: DotNet-Docker-Common
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-Docker-Secrets-WIF