Move testnet metrics dashboard management out of the Grafana UI

ci/publish-metrics-dashboard.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z $BUILDKITE ]]; then
+  echo BUILDKITE not defined
+  exit 1
+fi
+
+if [[ -z $CHANNEL ]]; then
+  CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
+fi
+
+if [[ -z $CHANNEL ]]; then
+  (
+    cat <<EOF
+steps:
+  - block: "Select Dashboard"
+    fields:
+      - select: "Channel"
+        key: "channel"
+        options:
+          - label: "stable"
+            value: "stable"
+          - label: "edge"
+            value: "edge"
+          - label: "beta"
+            value: "beta"
+  - command: "ci/$(basename "$0")"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+
+ci/channel-info.sh
+eval "$(ci/channel-info.sh)"
+
+case $CHANNEL in
+edge)
+  CHANNEL_BRANCH=$EDGE_CHANNEL
+  ;;
+beta)
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+stable)
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+  (
+    cat <<EOF
+steps:
+  - trigger: "$BUILDKITE_PIPELINE_SLUG"
+    async: true
+    build:
+      message: "$BUILDKITE_MESSAGE"
+      branch: "$CHANNEL_BRANCH"
+      env:
+        CHANNEL: "$CHANNEL"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+set -x
+exec metrics/publish-metrics-dashboard.sh "$CHANNEL"

metrics/README.md

@@ -0,0 +1,39 @@
+# Metrics
+
+## Testnet Grafana Dashboard
+
+There are three versions of the testnet dashboard, corresponding to the three
+release channels:
+* https://metrics.solana.com:3000/d/testnet-edge/testnet-monitor-edge
+* https://metrics.solana.com:3000/d/testnet-beta/testnet-monitor-beta
+* https://metrics.solana.com:3000/d/testnet/testnet-monitor
+
+The dashboard for each channel is defined from the
+`metrics/testnet-monitor.json` source file in the git branch associated with
+that channel, and deployed by automation running `ci/publish-metrics-dashboard.sh`.
+
+A deploy can be triggered at any time via the `New Build` button of
+https://buildkite.com/solana-labs/publish-metrics-dashboard.
+
+### Modifying a Dashboard
+
+Dashboard updates are accomplished by modifying `metrics/testnet-monitor.json`,
+**manual edits made directly in Grafana will be overwritten**.
+
+1. Open the desired dashboard in Grafana
+2. Create a development copy of the dashboard by selecting `Save As..` in the
+   `Settings` menu for the dashboard
+3. Edit dashboard as desired
+4. Extract the JSON Model by selecting `JSON Model` in the `Settings` menu.  Copy the JSON to the clipboard
+    and paste into `metrics/testnet-monitor.json`
+5. Delete your development dashboard: `Settings` => `Delete`
+
+### Deploying a Dashboard Manually
+
+If you need to immediately deploy a dashboard using the contents of
+`metrics/testnet-monitor.json` in your local workspace,
+```
+$ export GRAFANA_API_TOKEN="an API key from https://metrics.solana.com:3000/org/apikeys"
+$ metrics/publish-metrics-dashboard.sh (edge|beta|stable)
+```
+Note that automation will eventually overwrite your manual deploy.

metrics/adjust-dashboard-for-channel.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Adjusts the testnet monitor dashboard for the specified release channel
+#
+
+import sys
+import json
+
+if len(sys.argv) != 3:
+    print('Error: Dashboard or Channel not specified')
+    sys.exit(1)
+
+dashboard_json = sys.argv[1]
+channel = sys.argv[2]
+if channel not in ['edge', 'beta', 'stable']:
+    print('Error: Unknown channel:', channel)
+    sys.exit(2)
+
+with open(dashboard_json, 'r') as read_file:
+    data = json.load(read_file)
+
+if channel == 'stable':
+    # Stable dashboard only allows the user to select between the stable
+    # testnet databases
+    data['title'] = 'Testnet Monitor'
+    data['uid'] = 'testnet'
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [{'selected': False,
+                                                'text': 'testnet',
+                                                'value': 'testnet'},
+                                               {'selected': True,
+                                                'text': 'testnet-perf',
+                                                'value': 'testnet-perf'}],
+                                   'query': 'testnet,testnet-perf',
+                                   'type': 'custom'}]
+else:
+    # Non-stable dashboard only allows the user to select between all testnet
+    # databases
+    data['title'] = 'Testnet Monitor ({})'.format(channel)
+    data['uid'] = 'testnet-' + channel
+    data['templating']['list'] = [{'allValue': None,
+                                   'current': {'text': 'testnet',
+                                               'value': 'testnet'},
+                                   'datasource': 'Solana Metrics (read-only)',
+                                   'hide': 1,
+                                   'includeAll': False,
+                                   'label': 'Testnet',
+                                   'multi': False,
+                                   'name': 'testnet',
+                                   'options': [],
+                                   'query': 'show databases',
+                                   'refresh': 1,
+                                   'regex': 'testnet.*',
+                                   'sort': 1,
+                                   'tagValuesQuery': '',
+                                   'tags': [],
+                                   'tagsQuery': '',
+                                   'type': 'query',
+                                   'useTags': False}]
+
+with open(dashboard_json, 'w') as write_file:
+    json.dump(data, write_file, indent=2)

metrics/grafcli.conf

@@ -0,0 +1,15 @@
+[grafcli]
+editor = vim
+mergetool = vimdiff
+verbose = on
+force = on
+
+[resources]
+
+[hosts]
+metrics = on
+
+[metrics]
+type = api
+url = https://metrics.solana.com:3000/api
+ssl = off

metrics/publish-metrics-dashboard.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")"
+
+CHANNEL=$1
+if [[ -z $CHANNEL ]]; then
+  echo "usage: $0 [channel]"
+  exit 1
+fi
+
+case $CHANNEL in
+edge)
+  DASHBOARD=testnet-monitor-edge
+  ;;
+beta)
+  DASHBOARD=testnet-monitor-beta
+  ;;
+stable)
+  DASHBOARD=testnet-monitor
+  ;;
+*)
+  echo "Error: Invalid CHANNEL=$CHANNEL"
+  exit 1
+  ;;
+esac
+
+
+if [[ -z $GRAFANA_API_TOKEN ]]; then
+  echo Error: GRAFANA_API_TOKEN not defined
+  exit 1
+fi
+
+DASHBOARD_JSON=./testnet-monitor.json
+if [[ ! -r $DASHBOARD_JSON ]]; then
+  echo Error: $DASHBOARD_JSON not found
+fi
+
+(
+  set -x
+  ./adjust-dashboard-for-channel.py "$DASHBOARD_JSON" "$CHANNEL"
+)
+
+rm -rf venv
+python3 -m venv venv
+# shellcheck source=/dev/null
+source venv/bin/activate
+
+echo --- Fetch/build grafcli
+(
+  set -x
+  git clone git@github.com:mvines/grafcli.git -b experimental-v5 venv/grafcli
+  cd venv/grafcli
+  python3 setup.py install
+)
+
+echo --- Take a backup of existing dashboard if possible
+(
+  set -x +e
+  grafcli export remote/metrics/$DASHBOARD $DASHBOARD_JSON.org
+  grafcli rm remote/metrics/$DASHBOARD
+  :
+)
+
+echo --- Publish $DASHBOARD_JSON to $DASHBOARD
+(
+  set -x
+  grafcli import $DASHBOARD_JSON remote/metrics
+)
+
+exit 0