GHW for building and publishing docker images (#1391)

* add ghw and scripts for docker image deployment * debug * add permissions for content * fix path to the bin folder * add tags * rename env * fix path to docker file * make polkadot-parachain executable * fix typo * fix more typos * test * revert back use of working directory * mke bin executable in the artifacts folder * use cd instead of working directory * change path to cash * fix path to cash * change cache key * delete old flows * addressed PR comments * fix path * reorg docker files
paritytech · Sep 8, 2023 · 1be427e · 1be427e
1 parent 5e56b1a
commit 1be427e
Show file tree

Hide file tree

Showing 55 changed files with 784 additions and 677 deletions.
diff --git a/.gitlab/common/lib.sh → .github/scripts/common/lib.sh b/.gitlab/common/lib.sh → .github/scripts/common/lib.sh
@@ -96,7 +96,7 @@ structure_message() {
 # access_token: see https://matrix.org/docs/guides/client-server-api/
 # Usage: send_message $body (json formatted) $room_id $access_token
 send_message() {
-curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
+  curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
 }
 
 # Pretty-printing functions
@@ -193,3 +193,74 @@ check_bootnode(){
     echo "    Bootnode appears unreachable"
     return 1
 }
+
+# Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts() {
+  echo "Release ID : $RELEASE_ID"
+  echo "Repo       : $REPO"
+  echo "Binary     : $BINARY"
+
+  curl -L -s \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json
+
+  # Get Asset ids
+  ids=($(jq -r '.assets[].id' < release.json ))
+  count=$(jq '.assets|length' < release.json )
+
+  # Fetch artifacts
+  mkdir -p "./release-artifacts/${BINARY}"
+  pushd "./release-artifacts/${BINARY}" > /dev/null
+
+  iter=1
+  for id in "${ids[@]}"
+  do
+      echo " - $iter/$count: downloading asset id: $id..."
+      curl -s -OJ -L -H "Accept: application/octet-stream" \
+          -H "Authorization: Token ${GITHUB_TOKEN}" \
+          "https://api.github.com/repos/${REPO}/releases/assets/$id"
+      iter=$((iter + 1))
+  done
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+}
+
+# Check the checksum for a given binary
+function check_sha256() {
+    echo "Checking SHA256 for $1"
+    shasum -qc $1.sha256
+}
+
+# Import GPG keys of the release team members
+# This is done in parallel as it can take a while sometimes
+function import_gpg_keys() {
+  GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"}
+  SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798"
+  WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2"
+  EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
+  MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF"
+  MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
+
+  echo "Importing GPG keys from $GPG_KEYSERVER in parallel"
+  for key in $SEC $WILL $EGOR $MARA $MORGAN; do
+    (
+      echo "Importing GPG key $key"
+      gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
+      echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust;
+    ) &
+  done
+  wait
+}
+
+# Check the GPG signature for a given binary
+function check_gpg() {
+    echo "Checking GPG Signature for $1"
+    gpg --no-tty --verify -q $1.asc $1
+}
diff --git a/...b/workflows/release-50_publish-docker.yml → ...b/workflows/release-50_publish-docker.yml b/...b/workflows/release-50_publish-docker.yml → ...b/workflows/release-50_publish-docker.yml
@@ -7,9 +7,10 @@ name: Release - Publish Docker Image
 # image and publishes it.
 
 on:
-  release:
-    types:
-      - published
+  #TODO: activate automated run later
+  # release:
+  #  types:
+  #    - published
   workflow_dispatch:
     inputs:
       release_id:
@@ -39,6 +40,18 @@ on:
         required: true
         type: string
         default: parity
+      binary:
+        description: Binary to be published
+        required: true
+        default: polkadot
+        type: choice
+        options:
+          - polkadot
+          - staking-miner
+          - polkadot-parachain
+
+permissions:
+  contents: write
 
 env:
   RELEASE_ID: ${{ inputs.release_id }}
@@ -47,8 +60,8 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }}
   REPO: ${{ github.repository }}
-  BINARY: polkadot-parachain
-  EVENT_ACTION: ${{ github.event.action }}
+  BINARY: ${{ inputs.binary }}
+  # EVENT_ACTION: ${{ github.event.action }}
   EVENT_NAME: ${{ github.event_name }}
   IMAGE_TYPE: ${{ inputs.image_type }}
 
@@ -58,100 +71,68 @@ jobs:
 
     steps:
       - name: Checkout sources
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-
-      - name: Prepare temp folder
-        run: |
-          TMP=$(mktemp -d)
-          echo "TMP=$TMP" >> "$GITHUB_ENV"
-          pwd
-          ls -al "$TMP"
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
-      - name: Fetch lib.sh from polkadot repo
-        working-directory: ${{ env.TMP }}
-        run: |
-          curl -O -L \
-            -H "Accept: application/vnd.github.v3.raw" \
-            https://raw.githubusercontent.com/paritytech/polkadot/master/scripts/ci/common/lib.sh
-
-          chmod a+x lib.sh
-          ls -al
-
-      - name: Fetch release artifacts based on final release tag
+      #TODO: this step will be needed when automated triggering will work
         #this step runs only if the workflow is triggered automatically when new release is published
-        if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
-        run: |
-          mkdir -p release-artifacts && cd release-artifacts
-
-          for f in $BINARY $BINARY.asc $BINARY.sha256; do
-            URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f"
-            echo " - Fetching $f from $URL"
-            wget "$URL" -O "$f"
-          done
-          chmod a+x $BINARY
-          cp -f ${TMP}/lib.sh .
-          ls -al
+        # if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
+        # run: |
+        #   mkdir -p release-artifacts && cd release-artifacts
+
+        #   for f in $BINARY $BINARY.asc $BINARY.sha256; do
+        #     URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f"
+        #     echo " - Fetching $f from $URL"
+        #     wget "$URL" -O "$f"
+        #   done
+        #   chmod a+x $BINARY
+        #   ls -al
 
       - name: Fetch rc artifacts or release artifacts based on release id
         #this step runs only if the workflow is triggered manually
         if: ${{ env.EVENT_NAME  == 'workflow_dispatch' }}
         run: |
-          . ${TMP}/lib.sh
+          . ./.github/scripts/common/lib.sh
 
           fetch_release_artifacts
 
-          chmod a+x release-artifacts/$BINARY
-          ls -al
-
-          cp -f ${TMP}/lib.sh release-artifacts/
-
       - name: Cache the artifacts
         uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
         with:
-          key: artifacts-${{ github.sha }}
+          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
           path: |
-            ./release-artifacts/**/*
+            ./release-artifacts/${{ env.BINARY }}/**/*
 
   build-container:
     runs-on: ubuntu-latest
     needs: fetch-artifacts
 
     steps:
       - name: Checkout sources
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
       - name: Get artifacts from cache
         uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
         with:
-          key: artifacts-${{ github.sha }}
+          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
           fail-on-cache-miss: true
           path: |
-            ./release-artifacts/**/*
+            ./release-artifacts/${{ env.BINARY }}/**/*
 
       - name: Check sha256 ${{ env.BINARY }}
-        working-directory: ./release-artifacts
+        working-directory: ./release-artifacts/${{ env.BINARY }}
         run: |
-          . ./lib.sh
+          . ../../.github/scripts/common/lib.sh
 
           echo "Checking binary $BINARY"
           check_sha256 $BINARY && echo "OK" || echo "ERR"
 
       - name: Check GPG ${{ env.BINARY }}
-        working-directory: ./release-artifacts
+        working-directory: ./release-artifacts/${{ env.BINARY }}
         run: |
-          . ./lib.sh
+          . ../../.github/scripts/common/lib.sh
           import_gpg_keys
           check_gpg $BINARY
 
-      - name: Build Injected Container image for ${{ env.BINARY }}
-        env:
-          IMAGE_NAME: ${{ env.BINARY }}
-          OWNER: ${{ env.DOCKER_OWNER }}
-        run: |
-          ls -al
-          echo "Building container for $BINARY"
-          ./docker/scripts/build-injected-image.sh
-
       - name: Fetch rc commit and tag
         if: ${{ env.IMAGE_TYPE == 'rc' }}
         id: fetch_rc_refs
@@ -167,36 +148,55 @@ jobs:
           echo "No tag, doing without"
 
       - name: Fetch release tags
-        if: ${{ env.IMAGE_TYPE == 'release' || env.EVENT_NAME == 'release' &&  env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
+        working-directory: ./release-artifacts/${{ env.BINARY }}
+        if: ${{ env.IMAGE_TYPE == 'release'}}
         id: fetch_release_refs
         run: |
-          VERSION=$(docker run --pull never --rm $DOCKER_OWNER/$BINARY --version | awk '{ print $2 }' )
+          chmod a+rx $BINARY
+          VERSION=$(./$BINARY --version | awk '{ print $2 }' )
           release=$( echo $VERSION | cut -f1 -d- )
           echo "tag=latest" >> $GITHUB_OUTPUT
           echo "release=${release}" >> $GITHUB_OUTPUT
 
+      - name: Build Injected Container image for polkadot/staking-miner
+        if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'staking-miner' }}
+        env:
+          ARTIFACTS_FOLDER: ./release-artifacts
+          IMAGE_NAME: ${{ env.BINARY }}
+          OWNER: ${{ env.DOCKER_OWNER }}
+          TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
+        run: |
+          ls -al
+          echo "Building container for $BINARY"
+          ./docker/scripts/build-injected.sh
+
+      - name: Build Injected Container image for polkadot-parachain
+        if: ${{ env.BINARY == 'polkadot-parachain' }}
+        env:
+          ARTIFACTS_FOLDER: ./release-artifacts
+          IMAGE_NAME: ${{ env.BINARY }}
+          OWNER: ${{ env.DOCKER_OWNER }}
+          DOCKERFILE: docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile
+          TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
+        run: |
+          ls -al
+          mkdir -p $ARTIFACTS_FOLDER/specs
+          cp cumulus/parachains/chain-specs/*.json $ARTIFACTS_FOLDER/specs
+
+          echo "Building container for $BINARY"
+          ./docker/scripts/build-injected.sh
 
       - name: Login to Dockerhub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Tag and Push Container image for ${{ env.BINARY }}
+      - name: Push Container image for ${{ env.BINARY }}
         id: docker_push
-        env:
-          TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
         run: |
-          TAGS=${TAGS[@]:-latest}
-          IFS=',' read -r -a TAG_ARRAY <<< "$TAGS"
-
-          echo "The image ${BINARY} will be tagged with ${TAG_ARRAY[*]}"
-          for TAG in "${TAG_ARRAY[@]}"; do
-            $ENGINE tag ${DOCKER_OWNER}/${BINARY} ${DOCKER_OWNER}/${BINARY}:${TAG}
-            $ENGINE push ${DOCKER_OWNER}/${BINARY}:${TAG}
-          done
-
           $ENGINE images | grep ${BINARY}
+          $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY}
 
       - name: Check version for the published image for ${{ env.BINARY }}
         env:

diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml
@@ -328,7 +328,7 @@ build-linux-substrate:
       cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION;
       fi
     - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
-    - cp -r ./docker/substrate_injected.Dockerfile ./artifacts/substrate/
+    - cp -r ./docker/dockerfiles/substrate_injected.Dockerfile ./artifacts/substrate/
     # - printf '\n# building node-template\n\n'
     # - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz