[Ci] Support to trigger a pipeline to download and publish artifacts …

…to storage (#10820)

Why I did it
Support to trigger a pipeline to download and publish artifacts to storage and container registry.
Support to specify the patterns which docker images to upload.

How I did it
Pass the pipeline information and the artifact information by pipeline parameters to the pipeline which will be triggered a new build. It is to decouple the artifacts generation and the publish logic, how and where the artifacts/docker images will be published, depends on the triggered pipeline.

How to verify it

.azure-pipelines/azure-pipelines-image-template.yml

@@ -48,7 +48,7 @@ jobs:
           ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
         displayName: 'Make configure'
     postSteps:
-      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+      - script: mv target $(Build.ArtifactStagingDirectory)/
         displayName: Copy Artifacts
       - publish:  $(Build.ArtifactStagingDirectory)
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
@@ -57,6 +57,10 @@ jobs:
         condition: failed()
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)$(System.JobAttempt)'
         displayName: "Archive failed sonic image"
+      - template: trigger-publish-artifacts-build.yml
+        parameters:
+          artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+          publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
       - template: cleanup.yml
     jobGroups: ${{ parameters.jobGroups }}

.azure-pipelines/trigger-publish-artifacts-build.yml

@@ -0,0 +1,64 @@
+# The steps to trigger the pipeline to publish the artifacts
+
+parameters:
+- name: artifactName
+  type: string
+  default: ""
+- name: publishPrefix
+  type: string
+  default: "$(Build.DefinitionName)/$(Build.SourceBranchName)"
+
+steps:
+- script: |
+    . functions.sh
+    sonic_version=$(sonic_get_version)
+    latest_tag=$(git describe --tags --abbrev=0)
+    docker_tags="$sonic_version $(Build.SourceBranchName)"
+    if [ "$(Build.SourceBranchName)" == "master" ]; then
+      docker_tags="$docker_tags latest"
+    fi
+    echo "##vso[task.setvariable variable=sonic_version]$sonic_version"
+    echo "##vso[task.setvariable variable=latest_tag]$latest_tag"
+    echo "##vso[task.setvariable variable=docker_tags]$docker_tags"
+  condition: ne(variables['Build.Reason'], 'PullRequest')
+  displayName: 'Set trigger build variables'
+- task: TriggerBuild@4
+  condition: ne(variables['Build.Reason'], 'PullRequest')
+  inputs:
+    definitionIsInCurrentTeamProject: false
+    teamProject: internal
+    tfsServer: $(System.CollectionUri)
+    buildDefinition: 'publish-artifacts'
+    queueBuildForUserThatTriggeredBuild: true
+    ignoreSslCertificateErrors: false
+    useSameSourceVersion: false
+    useCustomSourceVersion: false
+    useSameBranch: false
+    waitForQueuedBuildsToFinish: false
+    storeInEnvironmentVariable: true
+    authenticationMethod: 'Personal Access Token'
+    password: '$(system.accesstoken)'
+    enableBuildInQueueCondition: false
+    dependentOnSuccessfulBuildCondition: false
+    dependentOnFailedBuildCondition: false
+    checkbuildsoncurrentbranch: false
+    failTaskIfConditionsAreNotFulfilled: false
+    buildParameters: ''
+    templateParameters: |
+      pipelineContext: {"buildId":"$(Build.BuildId)",
+        "pipelineId":"$(System.DefinitionId)",
+        "project": "$(System.TeamProject)",
+        "branchName":"$(Build.SourceBranchName)"},
+      artifactContext: {"artifactName":"${{ parameters.artifactName }}",
+        "artifactPatterns":"**/*.bin\n
+              **/*.swi\n
+              **/*.raw\n
+              **/*.img.gz\n
+              **/*-rpc.gz\n
+              **/python-saithrift*.deb"},
+      publishContext: {"publishPrefix":"${{ parameters.publishPrefix }}",
+        "keepArtifactName":false,
+        "dockerImagePatterns":"target/*-rpc.gz",
+        "dockerTags":"$(docker_tags)",
+        "version":"$(sonic_version)",
+        "latestTag":"$(latest_tag)"}