tweak OTEL alerts to use SR

CHANGELOG.md

@@ -20,6 +20,8 @@ Main (unreleased)
 
 - Small fix in UI stylesheet to fit more content into visible table area. (@defanator)
 
+- Changed OTEL alerts in Alloy mixin to use success rate for tracing. (@thampiotr)
+
 v1.4.0-rc.3
 -----------------
 

operations/alloy-mixin/alerts/opentelemetry.libsonnet

@@ -1,37 +1,40 @@
 local alert = import './utils/alert.jsonnet';
 
 {
+  local successRateQuery(enableK8sCluster, failed, success) =
+        local sumBy = if enableK8sCluster then "cluster, namespace, job" else "job";
+        |||
+          (1 - sum by (%s) (
+                  rate(%s{}[1m])
+                  /
+                  (rate(%s{}[1m]) + rate(%s{}[1m]))
+               )
+          ) < 0.95
+        ||| % [sumBy, failed, failed, success],
+
   newOpenTelemetryAlertsGroup(enableK8sCluster=true):
     alert.newGroup(
       'alloy_otelcol',
       [
-        // An otelcol.exporter component rcould not push some spans to the pipeline.
+        // An otelcol.receiver component could not push over 5% of spans to the pipeline.
         // This could be due to reaching a limit such as the ones
         // imposed by otelcol.processor.memory_limiter.
         alert.newRule(
           'OtelcolReceiverRefusedSpans',
-          if enableK8sCluster then
-            'sum by (cluster, namespace, job) (rate(otelcol_receiver_refused_spans_total{}[1m])) > 0'
-          else
-            'sum by (job) (rate(otelcol_receiver_refused_spans_total{}[1m])) > 0'
-          ,
-          'The receiver could not push some spans to the pipeline.',
+          successRateQuery(enableK8sCluster, "otelcol_receiver_refused_spans_total", "otelcol_receiver_accepted_spans_total"),
+          'The receiver pushing spans to the pipeline success rate is below 95%.',
           'The receiver could not push some spans to the pipeline under job {{ $labels.job }}. This could be due to reaching a limit such as the ones imposed by otelcol.processor.memory_limiter.',
-          '5m',
+          '10m',
         ),
 
-        // The exporter failed to send spans to their destination.
+        // The exporter success rate is below 95%.
         // There could be an issue with the payload or with the destination endpoint.
         alert.newRule(
           'OtelcolExporterFailedSpans',
-          if enableK8sCluster then
-            'sum by (cluster, namespace, job) (rate(otelcol_exporter_send_failed_spans_total{}[1m])) > 0'
-          else
-            'sum by (job) (rate(otelcol_exporter_send_failed_spans_total{}[1m])) > 0'
-          ,
-          'The exporter failed to send spans to their destination.',
+          successRateQuery(enableK8sCluster, "otelcol_exporter_send_failed_spans_total", "otelcol_exporter_sent_spans_total"),
+          'The exporter sending spans success rate is below 95%.',
           'The exporter failed to send spans to their destination under job {{ $labels.job }}. There could be an issue with the payload or with the destination endpoint.',
-          '5m',
+          '10m',
         ),
       ]
     )