Add extrincic ordering check to CI (#629)

* Add extrincic ordering check to CI

* Add missing script

* Fix default chain

* Ignore issues with pkill

* Continue on error while trying to stop the node

* Set embedded relay chain node

* Fix formatting

Author: chevdor

.github/workflows/extrinsic-ordering-check-from-bin.yml

@@ -0,0 +1,82 @@
+# This workflow performs the Extrinsic Ordering Check on demand using a binary
+
+name: Extrinsic Ordering Check from Binary
+on:
+  workflow_dispatch:
+    inputs:
+      reference_url:
+        description: The WebSocket url of the reference node
+        default: wss://kusama-statemine-rpc.paritytech.net
+        required: true
+      binary_url:
+        description: A url to a Linux binary for the node containing the runtime to test
+        default: https://github.com/paritytech/cumulus/releases/download/statemine_v3/polkadot-collator
+        required: true
+      chain:
+        description: The name of the chain under test. Usually, you would pass a local chain
+        default: statemine-local
+        required: true
+
+jobs:
+  check:
+    name: Run check
+    runs-on: ubuntu-latest
+    env:
+      CHAIN: ${{github.event.inputs.chain}}
+      BIN_URL: ${{github.event.inputs.binary_url}}
+      REF_URL: ${{github.event.inputs.reference_url}}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Fetch binary
+        run: |
+          echo Fetching $BIN_URL
+          wget $BIN_URL
+          chmod a+x polkadot-collator
+          ./polkadot-collator --version
+
+      - name: Start local node
+        run: |
+          echo Running on $CHAIN
+          ./polkadot-collator --chain=$CHAIN -- --chain polkadot-local &
+
+      - name: Prepare output
+        run: |
+          VERSION=$(./polkadot-collator --version)
+          echo "Metadata comparison:" >> output.txt
+          echo "Date: $(date)" >> output.txt
+          echo "Reference: $REF_URL" >> output.txt
+          echo "Target version: $VERSION" >> output.txt
+          echo "Chain: $CHAIN" >> output.txt
+          echo "----------------------------------------------------------------------" >> output.txt
+
+      - name: Pull polkadot-js-tools image
+        run: docker pull jacogr/polkadot-js-tools
+
+      - name: Compare the metadata
+        run: |
+          CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $REF_URL ws://localhost:9944"
+          echo -e "Running:\n$CMD"
+          $CMD >> output.txt
+          sed -z -i 's/\n\n/\n/g' output.txt
+          cat output.txt | egrep -n -i ''
+          SUMMARY=$(./scripts/extrinsic-ordering-filter.sh output.txt)
+          echo -e $SUMMARY
+          echo -e $SUMMARY >> output.txt
+
+      - name: Show result
+        run: |
+          cat output.txt
+
+      - name: Stop our local node
+        run: |
+          pkill polkadot-collator
+        continue-on-error: true
+
+      - name: Save output as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.CHAIN }}
+          path: |
+            output.txt

scripts/extrinsic-ordering-filter.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# This script is used in a Github Workflow. It helps filtering out what is interesting
+# when comparing metadata and spot what would require a tx version bump.
+
+# shellcheck disable=SC2002,SC2086
+
+FILE=$1
+
+# Higlight indexes that were deleted
+function find_deletions() {
+    echo "\n## Deletions\n"
+    RES=$(cat "$FILE" | grep -n '\[\-\]' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight indexes that have been deleted
+function find_index_changes() {
+    echo "\n## Index changes\n"
+    RES=$(cat "$FILE" | grep -E -n -i 'idx:\s*([0-9]+)\s*(->)\s*([0-9]+)' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight values that decreased
+function find_decreases() {
+    echo "\n## Decreases\n"
+    OUT=$(cat "$FILE" | grep -E -i -o '([0-9]+)\s*(->)\s*([0-9]+)' | awk '$1 > $3 { printf "%s;", $0 }')
+    IFS=$';' LIST=("$OUT")
+    unset RES
+    for line in "${LIST[@]}"; do
+        RES="$RES\n$(cat "$FILE" | grep -E -i -n \"$line\" | tr -s " ")"
+    done
+
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }' | sort -u -g | uniq
+    else
+        echo "n/a"
+    fi
+}
+
+echo "\n------------------------------ SUMMARY -------------------------------"
+echo "\n⚠️ This filter is here to help spotting changes that should be reviewed carefully."
+echo "\n⚠️ It catches only index changes, deletions and value decreases".
+
+find_deletions "$FILE"
+find_index_changes "$FILE"
+find_decreases "$FILE"
+echo "\n----------------------------------------------------------------------\n"