Import cortex mixin from upstream

.github/workflows/test-build-deploy.yml

@@ -36,6 +36,8 @@ jobs:
         run: make BUILD_IN_CONTAINER=false check-protos
       - name: Check Generated Documentation
         run: make BUILD_IN_CONTAINER=false check-doc
+      - name: Check Mixin
+        run: make BUILD_IN_CONTAINER=false check-mixin
       - name: Check White Noise.
         run: make BUILD_IN_CONTAINER=false check-white-noise
       - name: Check License Header

Makefile

@@ -2,7 +2,7 @@
 # WARNING: do not commit to a repository!
 -include Makefile.local
 
-.PHONY: all test integration-tests cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image license check-license format
+.PHONY: all test integration-tests cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image license check-license format check-mixin check-mixin-jb check-mixin-mixtool checkin-mixin-playbook build-mixin format-mixin
 .DEFAULT_GOAL := all
 
 # Version number
@@ -25,6 +25,12 @@ GIT_REVISION := $(shell git rev-parse --short HEAD)
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 UPTODATE := .uptodate
 
+# path to jsonnetfmt
+JSONNET_FMT := jsonnetfmt
+
+# path to the mimir/mixin
+MIXIN_PATH := jsonnet/mimir-mixin
+
 .PHONY: image-tag
 image-tag:
 	@echo $(IMAGE_TAG)
@@ -313,6 +319,33 @@ clean-white-noise:
 check-white-noise: clean-white-noise
 	@git diff --exit-code --quiet -- '*.md' || (echo "Please remove trailing whitespaces running 'make clean-white-noise'" && false)
 
+check-mixin: format-mixin check-mixin-jb check-mixin-mixtool check-mixin-playbook
+	@git diff --exit-code --quiet -- $(MIXIN_PATH) || (echo "Please format mixin by running 'make format-mixin'" && false)
+
+	@cd $(MIXIN_PATH) && \
+	jb install && \
+	mixtool lint mixin.libsonnet
+
+check-mixin-jb:
+	@cd $(MIXIN_PATH) && \
+	jb install
+
+check-mixin-mixtool: check-mixin-jb
+	@cd $(MIXIN_PATH) && \
+	mixtool lint mixin.libsonnet
+
+check-mixin-playbook: build-mixin
+	@$(MIXIN_PATH)/scripts/lint-playbooks.sh
+
+build-mixin: check-mixin-jb
+	@rm -rf $(MIXIN_PATH)/out && mkdir $(MIXIN_PATH)/out
+	@cd $(MIXIN_PATH) && \
+	mixtool generate all --output-alerts out/alerts.yaml --output-rules out/rules.yaml --directory out/dashboards mixin.libsonnet && \
+	zip -q -r mimir-mixin.zip out
+
+format-mixin:
+	@find $(MIXIN_PATH) -type f -name '*.libsonnet' -print -o -name '*.jsonnet' -print | xargs jsonnetfmt -i
+
 web-serve:
 	cd website && hugo --config config.toml --minify -v server
 

jsonnet/mimir-mixin/.gitignore

@@ -0,0 +1,3 @@
+/out/
+/vendor/
+/mimir-mixin.zip

jsonnet/mimir-mixin/README.md

@@ -0,0 +1,18 @@
+# Monitoring for Mimir
+
+To generate the Grafana dashboards and Prometheus alerts for Mimir:
+
+## Usage
+
+```console
+$ GO111MODULE=on go get github.com/monitoring-mixins/mixtool/cmd/mixtool
+$ GO111MODULE=on go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
+$ git clone https://github.com/grafana/mimir.git
+$ make build-mixin
+```
+
+This will leave all the alerts and dashboards in jsonnet/mimir-mixin/mimir-mixin.zip (or jsonnet/mimir-mixin/out).
+
+## Known Problems
+
+If you get an error like `cannot use cli.StringSliceFlag literal (type cli.StringSliceFlag) as type cli.Flag in slice literal` when installing [mixtool](https://github.com/monitoring-mixins/mixtool/issues/27), make sure you set `GO111MODULE=on` before `go get`.

jsonnet/mimir-mixin/alerts.libsonnet

@@ -0,0 +1,13 @@
+{
+  prometheusAlerts+::
+    (import 'alerts/alerts.libsonnet') +
+    (import 'alerts/alertmanager.libsonnet') +
+
+    (if std.member($._config.storage_engine, 'blocks')
+     then
+       (import 'alerts/blocks.libsonnet') +
+       (import 'alerts/compactor.libsonnet')
+     else {}) +
+
+    { _config:: $._config + $._group_config },
+}

jsonnet/mimir-mixin/alerts/alertmanager.libsonnet

@@ -0,0 +1,98 @@
+{
+  groups+: [
+    {
+      name: 'alertmanager_alerts',
+      rules: [
+        {
+          alert: 'CortexAlertmanagerSyncConfigsFailing',
+          expr: |||
+            rate(cortex_alertmanager_sync_configs_failed_total[5m]) > 0
+          |||,
+          'for': '30m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to read tenant configurations from storage.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexAlertmanagerRingCheckFailing',
+          expr: |||
+            rate(cortex_alertmanager_ring_check_errors_total[2m]) > 0
+          |||,
+          'for': '10m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} is unable to check tenants ownership via the ring.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexAlertmanagerPartialStateMergeFailing',
+          expr: |||
+            rate(cortex_alertmanager_partial_state_merges_failed_total[2m]) > 0
+          |||,
+          'for': '10m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to merge partial state changes received from a replica.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexAlertmanagerReplicationFailing',
+          expr: |||
+            rate(cortex_alertmanager_state_replication_failed_total[2m]) > 0
+          |||,
+          'for': '10m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to replicating partial state to its replicas.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexAlertmanagerPersistStateFailing',
+          expr: |||
+            rate(cortex_alertmanager_state_persist_failed_total[15m]) > 0
+          |||,
+          'for': '1h',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} is unable to persist full state snaphots to remote storage.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexAlertmanagerInitialSyncFailed',
+          expr: |||
+            increase(cortex_alertmanager_state_initial_sync_completed_total{outcome="failed"}[1m]) > 0
+          |||,
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Cortex Alertmanager {{ $labels.job }}/{{ $labels.instance }} was unable to obtain some initial state when starting up.
+            |||,
+          },
+        },
+      ],
+    },
+  ],
+}