[Release/CI] Github flow to build polkadot/polkadot-parachain rc …

…binaries and deb package (#5963) This PR introduces a GitHub flow, that should replace a semi manual part of the release process to build rc binaries for `polkadot` and `polkadot-parachain` + the `polkadot` deb package. Right now, this part of the release is done on the `cleanroom` machine by the release engineers via triggering bash scripts directly on the server. These GitHub flows should replace it and move everything to the CI. The whole flow is meant to be run in the new [paritytech-release](https://github.com/paritytech-release) where the automated release is going to be moved. The flow includes the following steps: - Build `polkadot`, `polakdot-prepare-worker`, `polkadot-execute-worker` and `polkadopt-parachain` binaries - Sign those artefacts using `gpg` and generate a sha256 checksum - Build deb package for `polakdot` - Make a GitHub attestation - Upload artefacts to the S3 buckets Closes: paritytech/release-engineering#223 --------- Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
paritytech · Oct 9, 2024 · c477076 · c477076
1 parent 6765bcd
commit c477076
Show file tree

Hide file tree

Showing 6 changed files with 390 additions and 0 deletions.
diff --git a/.github/scripts/release/build-deb.sh b/.github/scripts/release/build-deb.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+PRODUCT=$1
+VERSION=$2
+PROFILE=${PROFILE:-production}
+
+cargo install --version 2.7.0 cargo-deb --locked -q
+echo "Using cargo-deb v$(cargo-deb --version)"
+echo "Building a Debian package for '$PRODUCT' in '$PROFILE' profile"
+
+# we need to start  the custom version with a didgit as requires it cargo-deb
+cargo deb --profile $PROFILE --no-strip --no-build -p $PRODUCT --deb-version 1-$VERSION
+
+deb=target/debian/$PRODUCT_*_amd64.deb
+
+cp $deb target/production/
diff --git a/.github/scripts/release/build-linux-release.sh b/.github/scripts/release/build-linux-release.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+ARTIFACTS=/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/nul
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
diff --git a/.github/scripts/release/release_lib.sh b/.github/scripts/release/release_lib.sh
@@ -116,3 +116,24 @@ set_polkadot_parachain_binary_version() {
     commit_with_message "$MESSAGE"
     git_show_log "$MESSAGE"
 }
+
+
+upload_s3_release() {
+  alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+  product=$1
+  version=$2
+
+  echo "Working on product: $product "
+  echo "Working on version: $version "
+
+  echo "Current content, should be empty on new uploads:"
+  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize || true
+  echo "Content to be uploaded:"
+  artifacts="artifacts/$product/"
+  ls "$artifacts"
+  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/${version}/"
+  echo "Uploaded files:"
+  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize
+  echo "✅ The release should be at https://releases.parity.io/polkadot/${version}"
+}
diff --git a/.github/workflows/release-build-rc.yml b/.github/workflows/release-build-rc.yml
@@ -0,0 +1,74 @@
+name: Release - Build node release candidate
+
+on:
+  workflow_dispatch:
+    inputs:
+      binary:
+        description: Binary to be build for the release
+        default: all
+        type: choice
+        options:
+          - polkadot
+          - polkadot-parachain
+          - all
+
+      release_tag:
+        description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM
+        type: string
+
+jobs:
+  check-synchronization:
+    uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main
+
+  validate-inputs:
+    needs: [check-synchronization]
+    if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+        release_tag: ${{ steps.validate_inputs.outputs.release_tag }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Validate inputs
+        id: validate_inputs
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          RELEASE_TAG=$(validate_stable_tag ${{ inputs.release_tag }})
+          echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
+
+  build-polkadot-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot", "polkadot-prepare-worker", "polkadot-execute-worker"]'
+      package: polkadot
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+
+  build-polkadot-parachain-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-parachain"]'
+      package: "polkadot-parachain-bin"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/release-reusable-rc-buid.yml b/.github/workflows/release-reusable-rc-buid.yml
@@ -0,0 +1,191 @@
+name: RC Build
+
+on:
+  workflow_call:
+    inputs:
+      binary:
+        description: Binary to be build for the release
+        required: true
+        default: polkadot
+        type: string
+
+      package:
+        description: Package to be built, for now is either polkadot or polkadot-parachain-bin
+        required: true
+        type: string
+
+      release_tag:
+        description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM
+        required: true
+        type: string
+
+    secrets:
+      PGP_KMS_KEY:
+        required: true
+      PGP_KMS_HASH:
+        required: true
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      AWS_DEFAULT_REGION:
+        required: true
+      AWS_RELEASE_ACCESS_KEY_ID:
+        required: true
+      AWS_RELEASE_SECRET_ACCESS_KEY:
+        required: true
+
+permissions:
+  id-token: write
+  contents: read
+  attestations: write
+
+jobs:
+
+  set-image:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+
+  build-rc:
+    needs: [set-image]
+    runs-on: ubuntu-latest
+    environment: release
+    container:
+      image: ${{ needs.set-image.outputs.IMAGE }}
+    strategy:
+      matrix:
+        binaries: ${{ fromJSON(inputs.binary) }}
+    env:
+      PGP_KMS_KEY: ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH: ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+
+    steps:
+      - name: Install pgpkkms
+        run: |
+          # Install pgpkms that is used to sign built artifacts
+          python3 -m pip install "pgpkms @ git+https://github.com/paritytech-release/pgpkms.git@5a8f82fbb607ea102d8c178e761659de54c7af69"
+          which pgpkms
+
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          ref: ${{ inputs.release_tag }}
+          fetch-depth: 0
+
+      - name: Import gpg keys
+        shell: bash
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          import_gpg_keys
+
+      - name: Build binary
+        run: |
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}" #avoid "detected dubious ownership" error
+          ./.github/scripts/release/build-linux-release.sh ${{ matrix.binaries }} ${{ inputs.package }}
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
+        with:
+          subject-path: /artifacts/${{ matrix.binaries }}/${{ matrix.binaries }}
+
+      - name: Sign artifacts
+        working-directory: /artifacts/${{ matrix.binaries }}
+        run: |
+          python3 -m pgpkms sign --input ${{matrix.binaries }} -o ${{ matrix.binaries }}.asc
+
+      - name: Check sha256 ${{ matrix.binaries }}
+        working-directory: /artifacts/${{ matrix.binaries }}
+        shell: bash
+        run: |
+          .  "${GITHUB_WORKSPACE}"/.github/scripts/common/lib.sh
+
+          echo "Checking binary  ${{ matrix.binaries }}"
+          check_sha256  ${{ matrix.binaries }}
+
+      - name: Check GPG ${{ matrix.binaries }}
+        working-directory: /artifacts/${{ matrix.binaries }}
+        shell: bash
+        run: |
+          . "${GITHUB_WORKSPACE}"/.github/scripts/common/lib.sh
+
+          check_gpg  ${{ matrix.binaries }}
+
+      - name: Upload ${{ matrix.binaries }} artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ${{ matrix.binaries }}
+          path: /artifacts/${{ matrix.binaries }}
+
+  build-polkadot-deb-package:
+    if: ${{ inputs.package == 'polkadot' }}
+    needs: [build-rc]
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      with:
+        ref: ${{ inputs.release_tag }}
+        fetch-depth: 0
+
+    - name: Download artifacts
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        path: target/production
+        merge-multiple: true
+
+    - name: Build polkadot deb package
+      shell: bash
+      run: |
+        . "${GITHUB_WORKSPACE}"/.github/scripts/release/build-deb.sh ${{ inputs.package }} ${{ inputs.release_tag }}
+
+    - name: Generate artifact attestation
+      uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
+      with:
+        subject-path: target/production/*.deb
+
+    - name: Upload ${{inputs.package }} artifacts
+      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      with:
+        name: ${{ inputs.package }}
+        path: target/production
+        overwrite: true
+
+  upload-polkadot-artifacts-to-s3:
+    if: ${{ inputs.package == 'polkadot' }}
+    needs: [build-polkadot-deb-package]
+    uses: ./.github/workflows/release-reusable-s3-upload.yml
+    with:
+      package: ${{ inputs.package }}
+      release_tag: ${{ inputs.release_tag }}
+    secrets:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+
+
+  upload-polkadot-parachain-artifacts-to-s3:
+    if: ${{ inputs.package == 'polkadot-parachain-bin' }}
+    needs: [build-rc]
+    uses: ./.github/workflows/release-reusable-s3-upload.yml
+    with:
+      package: ${{ inputs.binary }}
+      release_tag: ${{ inputs.release_tag }}
+    secrets:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/release-reusable-s3-upload.yml b/.github/workflows/release-reusable-s3-upload.yml
@@ -0,0 +1,53 @@
+name: Upload to s3
+
+on:
+  workflow_call:
+    inputs:
+      package:
+        description: Package to be built, for now is either polkadot or polkadot-parachain-bin
+        required: true
+        type: string
+
+      release_tag:
+        description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM-rcX
+        required: true
+        type: string
+
+    secrets:
+      AWS_DEFAULT_REGION:
+        required: true
+      AWS_RELEASE_ACCESS_KEY_ID:
+        required: true
+      AWS_RELEASE_SECRET_ACCESS_KEY:
+        required: true
+
+jobs:
+  upload-artifacts-to-s3:
+      runs-on: ubuntu-latest
+      environment: release
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+        AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+
+      steps:
+        - name: Checkout
+          uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+        - name: Download artifacts
+          uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+          with:
+            name: ${{ inputs.package }}
+            path: artifacts/${{ inputs.package }}
+
+        - name: Configure AWS Credentials
+          uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+          with:
+            aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+            aws-region: ${{ env.AWS_REGION }}
+
+        - name: Upload ${{ inputs.package }} artifacts to s3
+          run: |
+            . ./.github/scripts/release/release_lib.sh
+            upload_s3_release ${{ inputs.package }} ${{ inputs.release_tag }}