Improve error rate and relayer activity metrics/alerts (#197)

CoreumFoundation · Mar 18, 2024 · db4e2c4 · db4e2c4
1 parent 7e8327a
commit db4e2c4
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/infra/composer/grafana/dashboards/relayer.json b/infra/composer/grafana/dashboards/relayer.json
@@ -1293,15 +1293,15 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "increase(relayer_errors_total[$__range])",
+          "expr": "increase(relayer_errors_total[5m])",
           "format": "time_series",
           "instant": false,
           "legendFormat": "Errors",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Errors count",
+      "title": "Errors increase 5m",
       "type": "timeseries"
     },
     {

diff --git a/infra/composer/prometheus/alert.rules b/infra/composer/prometheus/alert.rules
@@ -10,13 +10,13 @@ groups:
       labels:
         severity: "critical"
 
-    - alert: High relayer error rate
-      expr: rate(relayer_errors_total[5m]) * 5 * 60 > 10
-      for: 30s
+    - alert: High relayer error increase
+      expr: increase(relayer_errors_total[5m]) > 10
+      for: 10m
       labels:
         severity: major
       annotations:
-        description: "High relayer error rate"
+        description: "High relayer error increase"
 
     - alert: Low contract XRPL base fee
       expr: contract_config_xrpl_base_fee < xrpl_chain_base_fee
@@ -76,7 +76,7 @@ groups:
         description: "Detected malicious behaviour: {{ $labels.malicious_behaviour_key }}"
 
     - alert: No relayer activity for more than 24h
-      expr: increase(relayer_activity{action=~"save_evidence|save_signature"}[24h]) == 0
+      expr: (relayer_activity{action="save_evidence"} + on (relayer_coreum_address) relayer_activity{action="save_signature"}) == 0
       for: 10m # to let the relayer provide the metric after the restart
       labels:
         severity: "major"