Merge pull request sonic-net#50 from oleksandrivantsiv/bluefield-merge

Merge upstream/master into bluefield branch

Author: oleksandrivantsiv

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -18,6 +18,14 @@ schedules:
     - 202006
   always: true
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 pool: sonicbld
 
 parameters:
@@ -30,22 +38,24 @@ parameters:
   - centec
   - centec-arm64
   - generic
-  - innovium
   - marvell-armhf
   - mellanox
 
 stages:
 - stage: Build
   variables:
-    CACHE_MODE: none
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: CACHE_MODE
+      value: none
+    - name: VERSION_CONTROL_OPTIONS
+      value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
       jobFilters: ${{ parameters.jobFilters }}
-      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
-      - template: template-clean-sonic-slave.yml
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions
@@ -69,14 +79,14 @@ stages:
         default_platform=broadcom
         artifacts=$(find $(Pipeline.Workspace) -maxdepth 1 -type d -name 'sonic-buildimage.*' | grep -v "sonic-buildimage.${default_platform}")
         echo "artifacts$artifacts"
-        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
+        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/target/versions target/
         make freeze FREEZE_VERSION_OPTIONS=-r
         find files/build/versions 
         ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
         for artifact in $ordered_artifacts
         do
           rm -rf target/versions
-          cp -r $artifact/versions target/
+          cp -r $artifact/target/versions target/
           OPTIONS="-a -d"
           [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
           make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"

.azure-pipelines/azure-pipelines-build.yml

@@ -50,6 +50,7 @@ jobs:
             swi_image: yes
 
         - name: broadcom
+          timeoutInMinutes: 1440
           variables:
             dbg_image: yes
             swi_image: yes
@@ -131,3 +132,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-download-certificate.yml

@@ -0,0 +1,33 @@
+parameters:
+- name: connectionName
+  type: string
+  default: sonic-dev-connection
+- name: kevaultName
+  type: string
+  default: sonic-kv
+- name: certificateName
+  type: string
+  default: sonic-secure-boot
+
+steps:
+- task: AzureKeyVault@2
+  inputs:
+    connectedServiceName: ${{ parameters.connectionName }}
+    keyVaultName: ${{ parameters.kevaultName }}
+    secretsFilter: ${{ parameters.certificateName }}
+
+- script: |
+    set -e
+    TMP_FILE=$(mktemp)
+    echo "$CERTIFICATE" | base64 -d > $TMP_FILE
+    sudo mkdir -p /etc/certificates
+    mkdir -p $(Build.StagingDirectory)/target
+    # Save the public key
+    openssl pkcs12 -in $TMP_FILE -clcerts --nokeys -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN CERTIFICATE\)/\1/" > $(SIGNING_CERT)
+    # Save the private key
+    openssl pkcs12 -in $TMP_FILE -nocerts -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN PRIVATE KEY\)/\1/" | sudo tee $(SIGNING_KEY) 1>/dev/null
+    ls -lt $(SIGNING_CERT) $(SIGNING_KEY)
+    rm $TMP_FILE
+  env:
+    CERTIFICATE: $(${{ parameters.certificateName }})
+  displayName: "Save certificate"

.azure-pipelines/azure-pipelines-image-template.yml

@@ -48,16 +48,23 @@ jobs:
           ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
         displayName: 'Make configure'
     postSteps:
-      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+      - script: |
+          mkdir -p $(Build.ArtifactStagingDirectory)/target
+          mv target/* $(Build.ArtifactStagingDirectory)/target/
         displayName: Copy Artifacts
         condition: always()
       - publish:  $(Build.ArtifactStagingDirectory)
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
         displayName: "Archive sonic image"
       - publish:  $(Build.ArtifactStagingDirectory)
-        condition: failed()
+        condition: or(failed(), canceled())
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)$(System.JobAttempt)'
         displayName: "Archive failed sonic image"
+      - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        - template: trigger-publish-artifacts-build.yml
+          parameters:
+            artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+            publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
       - template: cleanup.yml
     jobGroups: ${{ parameters.jobGroups }}

.azure-pipelines/build-commonlib.yml

@@ -0,0 +1,19 @@
+pr: none
+trigger: none
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+jobs:
+- template: .azure-pipelines/template-commonlib.yml@buildimage

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-slave-arm64.yml

@@ -3,6 +3,13 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
 
 schedules:
 - cron: "0 8 * * *"
@@ -23,6 +30,7 @@ pr:
     - sonic-slave-stretch
     - sonic-slave-buster
     - sonic-slave-bullseye
+    - .azure-pipelines
 
 parameters:
 - name: 'dists'

.azure-pipelines/docker-sonic-slave-armhf.yml

@@ -3,6 +3,13 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
 
 schedules:
 - cron: "0 8 * * *"
@@ -23,6 +30,7 @@ pr:
     - sonic-slave-stretch
     - sonic-slave-buster
     - sonic-slave-bullseye
+    - .azure-pipelines
 
 parameters:
 - name: 'dists'

.azure-pipelines/docker-sonic-slave-template.yml

@@ -3,7 +3,6 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
-
 parameters:
 - name: arch
   type: string
@@ -38,7 +37,10 @@ jobs:
   pool: ${{ parameters.pool }}
   steps:
   - template: cleanup.yml
-  - template: template-clean-sonic-slave.yml
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+    - template: template-clean-sonic-slave.yml
+  - ${{ else }}:
+    - template: '/.azure-pipelines/template-clean-sonic-slave.yml@buildimage'
   - checkout: self
     clean: true
     submodules: recursive
@@ -81,6 +83,10 @@ jobs:
 
       docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
       docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
+      if [ "$SLAVE_BASE_IMAGE_UPLOAD" != "$SLAVE_DIR" ]; then
+        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:latest-${{ parameters.arch }}
+        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:$SLAVE_BASE_TAG
+      fi
       set +x
       echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
       echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
@@ -89,11 +95,26 @@ jobs:
     displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
 
   - task: Docker@2
+    condition: ne(variables['Build.Reason'], 'PullRequest')
     displayName: Upload image
     inputs:
       containerRegistry: ${{ parameters.registry_conn }}
       repository: $(VARIABLE_SLAVE_BASE_IMAGE)
       command: push
-      tags: |
-        $(VARIABLE_SLAVE_BASE_TAG)
-        latest
+      ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG)
+          latest
+      ${{ else }}:
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG)
+  - ${{ if ne(parameters.arch, 'amd64') }}:
+    - task: Docker@2
+      condition: ne(variables['Build.Reason'], 'PullRequest')
+      displayName: Upload image ${{ parameters.dist }}
+      inputs:
+        containerRegistry: ${{ parameters.registry_conn }}
+        repository: "sonic-slave-${{ parameters.dist }}"
+        command: push
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG) 

.azure-pipelines/docker-sonic-slave.yml

@@ -3,6 +3,13 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
 
 schedules:
 - cron: "0 8 * * *"
@@ -24,6 +31,7 @@ pr:
     - sonic-slave-buster
     - sonic-slave-bullseye
     - src/sonic-build-hooks
+    - .azure-pipelines
 
 parameters:
 - name: 'arches'
@@ -52,8 +60,15 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - template: docker-sonic-slave-template.yml
-          parameters:
-            pool: sonicbld
-            arch: ${{ arch }}
-            dist: ${{ dist }}
+        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+          - template: docker-sonic-slave-template.yml
+            parameters:
+              pool: sonicbld
+              arch: ${{ arch }}
+              dist: ${{ dist }}
+        - ${{ else }}:
+          - template: '/.azure-pipelines/docker-sonic-slave-template.yml@buildimage'
+            parameters:
+              pool: sonicbld
+              arch: ${{ arch }}
+              dist: ${{ dist }}

.azure-pipelines/official-build-cisco-8000.yml

@@ -22,10 +22,17 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+
 variables:
 - group: SONIC-AKV-STROAGE-1
 - name: StorageSASKey
   value: $(sonicstorage-SasToken)
+- name: SONIC_ENABLE_SECUREBOOT_SIGNATURE
+  value: y
+- name: SIGNING_KEY
+  value: /etc/certificates/sonic-secure-boot-private.pem
+- name: SIGNING_CERT
+  value: $(Build.StagingDirectory)/target/sonic-secure-boot-public.pem
 
 stages:
 - stage: Build
@@ -41,6 +48,7 @@ stages:
     parameters:
       buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
       preSteps:
+      - template: azure-pipelines-download-certificate.yml
       - checkout: self
         submodules: recursive
         path: s
@@ -90,5 +98,10 @@ stages:
           StorageSASKey: $(StorageSASKey)
         condition: ne(variables['Build.Reason'], 'PullRequest')
         displayName: "Override cisco sai packages"
+      - script: |
+          echo "SONIC_ENABLE_SECUREBOOT_SIGNATURE := y" >> rules/config.user
+          echo "SIGNING_KEY := $(SIGNING_KEY)" >> rules/config.user
+          echo "SIGNING_CERT := $(SIGNING_CERT)" >> rules/config.user
+        displayName: "Enable secure boot signature"
       jobGroups:
       - name: cisco-8000

.azure-pipelines/official-build.yml

@@ -18,9 +18,20 @@ schedules:
     - 201911
     - 201811
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 trigger: none
 pr: none
 
+variables:
+- template: .azure-pipelines/template-variables.yml@buildimage
+
 stages:
 - stage: Build
   pool: sonicbld