From 3f0e44481a83949dfd8b76f5c866470ca32295e3 Mon Sep 17 00:00:00 2001
From: s3krit <pugh@s3kr.it>
Date: Thu, 9 Apr 2020 12:58:28 +0200
Subject: [PATCH] CI: Release - monitor gitlab pipeline (#5460)

* Initial commit of gitlab pipeline monitoring

* increase max job time

* Improve debug output

* s/PR/issue/

Co-authored-by: Benjamin Kampmann <ben@parity.io>
---
 .github/ISSUE_TEMPLATE/release.md           |  9 +++++
 .github/workflows/check-gitlab-pipeline.yml | 30 +++++++++++++++++
 .maintain/github/check_gitlab_pipeline.sh   | 37 +++++++++++++++++++++
 3 files changed, 76 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/release.md
 create mode 100644 .github/workflows/check-gitlab-pipeline.yml
 create mode 100755 .maintain/github/check_gitlab_pipeline.sh

diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md
new file mode 100644
index 000000000000..6067dbf12fa7
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/release.md
@@ -0,0 +1,9 @@
+---
+title: Release failure for {{ ref }}
+---
+
+Pipeline for release {{ ref }} failed. Please investigate.
+
+If the pipeline has failed before pushing to crates.io, delete the release tag
+and fix the release as necessary, retagging after complete. If the pipeline has
+failed after pushing to crates.io, create a new tag incrementing the version.
diff --git a/.github/workflows/check-gitlab-pipeline.yml b/.github/workflows/check-gitlab-pipeline.yml
new file mode 100644
index 000000000000..c87f17c2f732
--- /dev/null
+++ b/.github/workflows/check-gitlab-pipeline.yml
@@ -0,0 +1,30 @@
+# A github action to track the status of the gitlab pipeline for tagged
+# releases, and cancel the release/create a new issue if it fails
+
+name: Monitor gitlab pipeline status
+
+on:
+  push:
+    tags:
+      - v*
+      - ci-release-*
+
+jobs:
+  monitor:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Monitor pipeline
+        run: env; ./.maintain/github/check_gitlab_pipeline.sh
+        id: monitor_pipeline
+        env:
+          TAGGER: ${{ github.event.pusher.name }}
+      - name: Create Issue
+        if: failure()
+        uses: JasonEtco/create-an-issue@v2
+        with:
+          filename: .github/ISSUE_TEMPLATE/release.md
+          assignees: ${{ github.event.pusher.name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.maintain/github/check_gitlab_pipeline.sh b/.maintain/github/check_gitlab_pipeline.sh
new file mode 100755
index 000000000000..e7aaff15bf6a
--- /dev/null
+++ b/.maintain/github/check_gitlab_pipeline.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+SUBSTRATE_API_BASEURL="https://gitlab.parity.io/api/v4/projects/145"
+
+TAG_NAME=$(echo "$GITHUB_REF" | sed -E 's_refs/tags/(.*)_\1_')
+PIPELINE_ID=$(curl -s $SUBSTRATE_API_BASEURL/pipelines | jq -r "map(select(.ref==\"$TAG_NAME\")) | .[0] | .id")
+if [ "$PIPELINE_ID" == "null" ]; then
+  echo "[!] Pipeline for $TAG_NAME not found. Exiting."
+  exit 1
+fi
+
+echo "[+] Pipeline path: https://gitlab.parity.io/parity/substrate/pipelines/$PIPELINE_ID"
+
+# 130 minute job max
+for (( c=0; c < 130; c++ )); do
+  out=$(curl -s "$SUBSTRATE_API_BASEURL/pipelines/$PIPELINE_ID" | jq -r .status)
+  case $out in
+    "success")
+      echo "[+] Pipeline $PIPELINE_ID for $TAG_NAME succeeded!"
+      exit 0
+      ;;
+    "failed")
+      echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME failed. Cannot proceed. Check job output on gitlab!"
+      exit 1
+      ;;
+    "cancelled")
+      echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME was cancelled. Cannot proceed!"
+      exit 1
+      ;;
+    "running")
+      echo "[-] Pipeline $PIPELINE_ID for $TAG_NAME still in progress..."
+  esac
+  sleep 60
+done
+# If we reach here, we timed out after 30 minutes
+echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME timed out! Cannot proceed"
+echo "::set-output name=pipeline_status::timedout"
+exit 1