Publish RC container images

.github/workflows/check-licenses.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
       - uses: actions/setup-node@v3.7.0
         with:
           node-version: '18.x'

.github/workflows/release-40_publish-rc-image.yml

@@ -0,0 +1,131 @@
+name: Release - Publish RC Container image
+# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_id:
+        description: |
+          Release ID.
+          You can find it using the command:
+            curl -s \
+              -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \
+              jq '.[] | { name: .name, id: .id }'
+        required: true
+        type: string
+      registry:
+        description: "Container registry"
+        required: true
+        type: string
+        default: docker.io
+      owner:
+        description: Owner of the container image repo
+        required: true
+        type: string
+        default: parity
+
+env:
+  RELEASE_ID: ${{ inputs.release_id }}
+  ENGINE: docker
+  REGISTRY: ${{ inputs.registry }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }}
+  REPO: ${{ github.repository }}
+
+jobs:
+  fetch-artifacts:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Fetch all artifacts
+        run: |
+          . ./scripts/ci/common/lib.sh
+          fetch_release_artifacts
+
+      - name: Cache the artifacts
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ github.sha }}
+          path: |
+            artifacts/**/*
+
+  build-container:
+    runs-on: ubuntu-latest
+    needs: fetch-artifacts
+
+    strategy:
+      matrix:
+        binary: ["polkadot", "staking-miner"]
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Get artifacts from cache
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ github.sha }}
+          path: |
+            artifacts/**/*
+
+      - name: Check sha256 ${{ matrix.binary }}
+        working-directory: ./artifacts
+        run: |
+          . ../scripts/ci/common/lib.sh
+
+          echo "Checking binary ${{ matrix.binary }}"
+          check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR"
+
+      - name: Check GPG ${{ matrix.binary }}
+        working-directory: ./artifacts
+        run: |
+          . ../scripts/ci/common/lib.sh
+          import_gpg_keys
+          check_gpg ${{ matrix.binary }}
+
+      - name: Fetch commit and tag
+        id: fetch_refs
+        run: |
+          release=release-${{ inputs.release_id }} && \
+          echo "release=${release}" >> $GITHUB_OUTPUT
+
+          commit=$(git rev-parse --short HEAD) && \
+          echo "commit=${commit}" >> $GITHUB_OUTPUT
+
+          tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \
+          [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \
+          echo "No tag, doing without"
+
+      - name: Build Injected Container image for ${{ matrix.binary }}
+        env:
+            BIN_FOLDER: artifacts
+            BINARY: ${{ matrix.binary }}
+            TAGS: ${{join(steps.fetch_refs.outputs.*, ',')}}
+        run: |
+          echo "Building container for ${{ matrix.binary }}"
+          ./scripts/ci/dockerfiles/build-injected.sh
+
+      - name: Login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ inputs.owner }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push Container image for ${{ matrix.binary }}
+        id: docker_push
+        env:
+          BINARY: ${{ matrix.binary }}
+        run: |
+          $ENGINE images | grep ${BINARY}
+          $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY}
+
+      - name: Check version for the published image for ${{ matrix.binary }}
+        env:
+          BINARY: ${{ matrix.binary }}
+          RELEASE_TAG: ${{ steps.fetch_refs.outputs.release }}
+        run: |
+          echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}"
+          $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version

.github/workflows/release-50_publish-docker-release.yml

@@ -30,7 +30,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.release.tag_name }}

.github/workflows/release-51_publish-docker-manual.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.inputs.version }}

.gitignore

@@ -10,3 +10,6 @@ polkadot.*
 !polkadot.service
 .DS_Store
 .env
+
+artifacts
+release.json

scripts/ci/common/lib.sh

@@ -193,3 +193,72 @@ check_bootnode(){
     echo "    Bootnode appears unreachable"
     return 1
 }
+
+# Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts() {
+  echo "Release ID : $RELEASE_ID"
+  echo "Repo       : $REPO"
+
+  curl -L -s \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/${REPO}/releases/$RELEASE_ID > release.json
+
+  # Get Asset ids
+  ids=($(jq -r '.assets[].id' < release.json ))
+  count=$(jq '.assets|length' < release.json )
+
+  # Fetch artifacts
+  mkdir -p artifacts
+  pushd artifacts > /dev/null
+
+  iter=1
+  for id in "${ids[@]}"
+  do
+      echo " - $iter/$count: downloading asset id: $id..."
+      curl -s -OJ -L -H "Accept: application/octet-stream" \
+          -H "Authorization: Token ${GITHUB_TOKEN}" \
+          "https://api.github.com/repos/${REPO}/releases/assets/$id"
+      iter=$((iter + 1))
+  done
+
+  ls -al --color
+  popd > /dev/null
+}
+
+# Check the checksum for a given binary
+function check_sha256() {
+    echo "Checking SHA256 for $1"
+    shasum -qc $1.sha256
+}
+
+# Import GPG keys of the release team members
+# This is done in parallel as it can take a while sometimes
+function import_gpg_keys() {
+  GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"}
+  SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798"
+  WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2"
+  EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
+  MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF"
+  MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
+
+  echo "Importing GPG keys from $GPG_KEYSERVER in parallel"
+  for key in $SEC $WILL $EGOR $MARA $MORGAN; do
+    (
+      echo "Importing GPG key $key"
+      gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
+      echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust;
+    ) &
+  done
+  wait
+}
+
+# Check the GPG signature for a given binary
+function check_gpg() {
+    echo "Checking GPG Signature for $1"
+    gpg --no-tty --verify -q $1.asc $1
+}

scripts/ci/dockerfiles/adder-collator/build-injected.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=adder-collator,undying-collator
+export BIN_FOLDER=$1
+
+$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh

scripts/ci/dockerfiles/adder-collator/test-build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+# TODO: Switch to /bin/bash when the image is built from parity/base-bin
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which adder-collator)" /export'
+
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which undying-collator)" /export'
+
+./build-injected.sh $TMP

scripts/ci/dockerfiles/binary_injected.Dockerfile

@@ -0,0 +1,48 @@
+FROM docker.io/parity/base-bin
+
+# This file allows building a Generic container image
+# based on one or multiple pre-built Linux binaries.
+# Some defaults are set to polkadot but all can be overriden.
+
+SHELL ["/bin/bash", "-c"]
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+ARG IMAGE_NAME
+
+# That can be a single one or a comma separated list
+ARG BINARY=polkadot
+
+ARG BIN_FOLDER=.
+ARG DOC_URL=https://github.com/paritytech/polkadot
+ARG DESCRIPTION="Polkadot: a platform for web3"
+ARG AUTHORS="devops-team@parity.io"
+ARG VENDOR="Parity Technologies"
+
+LABEL io.parity.image.authors=${AUTHORS} \
+	io.parity.image.vendor="${VENDOR}" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.title="${IMAGE_NAME}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="${DOC_URL}" \
+	io.parity.image.description="${DESCRIPTION}" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/binary_injected.Dockerfile"
+
+USER root
+WORKDIR /app
+
+# add polkadot binary to docker image
+# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/
+COPY entrypoint.sh .
+COPY "bin/*" "/usr/local/bin/"
+RUN chmod -R a+rx "/usr/local/bin"
+
+USER parity
+ENV BINARY=${BINARY}
+
+# ENTRYPOINT
+ENTRYPOINT ["/app/entrypoint.sh"]
+
+# We call the help by default
+CMD ["--help"]