From 0c0ca3967a67c0f95c4fa6ecb13bae3d1ed91dc5 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Tue, 27 Aug 2024 17:32:49 +0200 Subject: [PATCH 1/9] Introduce noLogsDelivered alert rule --- internal/selfmonitor/config/expr_builder.go | 5 +++ .../config/fluent_bit_rule_builder.go | 38 +++++++++++++------ internal/selfmonitor/config/rules.go | 6 ++- .../selfmonitor/prober/log_pipeline_prober.go | 11 ++---- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/internal/selfmonitor/config/expr_builder.go b/internal/selfmonitor/config/expr_builder.go index 0d17c1139..93960a798 100644 --- a/internal/selfmonitor/config/expr_builder.go +++ b/internal/selfmonitor/config/expr_builder.go @@ -78,6 +78,11 @@ func (eb *exprBuilder) greaterThan(value float64) *exprBuilder { return eb } +func (eb *exprBuilder) equal(value float64) *exprBuilder { + eb.expr = fmt.Sprintf("%s == %s", eb.expr, strconv.FormatFloat(value, 'f', -1, 64)) + return eb +} + func (eb *exprBuilder) build() string { return eb.expr } diff --git a/internal/selfmonitor/config/fluent_bit_rule_builder.go b/internal/selfmonitor/config/fluent_bit_rule_builder.go index e07deda5c..c87998303 100644 --- a/internal/selfmonitor/config/fluent_bit_rule_builder.go +++ b/internal/selfmonitor/config/fluent_bit_rule_builder.go @@ -1,5 +1,10 @@ package config +import ( + "fmt" + "time" +) + const ( fluentBitMetricsServiceName = "telemetry-fluent-bit-metrics" fluentBitSidecarMetricsServiceName = "telemetry-fluent-bit-exporter-metrics" @@ -19,10 +24,10 @@ type fluentBitRuleBuilder struct { func (rb fluentBitRuleBuilder) rules() []Rule { return []Rule{ rb.exporterSentRule(), - rb.receiverReadRule(), rb.exporterDroppedRule(), rb.bufferInUseRule(), rb.bufferFullRule(), + rb.noLogsDeliveredRule(), } } @@ -36,16 +41,6 @@ func (rb fluentBitRuleBuilder) exporterSentRule() Rule { } } -func (rb fluentBitRuleBuilder) receiverReadRule() Rule { - return Rule{ - Alert: rb.namePrefix() + RuleNameLogAgentReceiverReadLogs, - Expr: rate(metricFluentBitInputBytesTotal, selectService(fluentBitMetricsServiceName)). - sumBy(labelPipelineName). - greaterThan(0). - build(), - } -} - func (rb fluentBitRuleBuilder) exporterDroppedRule() Rule { return Rule{ Alert: rb.namePrefix() + RuleNameLogAgentExporterDroppedLogs, @@ -74,6 +69,27 @@ func (rb fluentBitRuleBuilder) bufferFullRule() Rule { } } +func (rb fluentBitRuleBuilder) noLogsDeliveredRule() Rule { + exporterNotSentExpr := rate(metricFluentBitOutputProcBytesTotal, selectService(fluentBitMetricsServiceName)). + sumBy(labelPipelineName). + equal(0). + build() + + receiverReadExpr := rate(metricFluentBitInputBytesTotal, selectService(fluentBitMetricsServiceName)). + sumBy(labelPipelineName). + greaterThan(0). + build() + + return Rule{ + Alert: rb.namePrefix() + RuleNameLogAgentNoLogsDelivered, + Expr: fmt.Sprintf(` +%s +and +%s`, exporterNotSentExpr, receiverReadExpr), + For: 3 * time.Minute, + } +} + func (rb fluentBitRuleBuilder) namePrefix() string { return ruleNamePrefix(typeLogPipeline) } diff --git a/internal/selfmonitor/config/rules.go b/internal/selfmonitor/config/rules.go index 282707b0b..6e8597e1d 100644 --- a/internal/selfmonitor/config/rules.go +++ b/internal/selfmonitor/config/rules.go @@ -17,10 +17,10 @@ const ( // Fluent Bit rule names. Note that the actual full names will be prefixed with Log RuleNameLogAgentExporterSentLogs = "AgentExporterSentLogs" - RuleNameLogAgentReceiverReadLogs = "AgentReceiverReadLogs" RuleNameLogAgentExporterDroppedLogs = "AgentExporterDroppedLogs" RuleNameLogAgentBufferInUse = "AgentBufferInUse" RuleNameLogAgentBufferFull = "AgentBufferFull" + RuleNameLogAgentNoLogsDelivered = "AgentNoLogsDelivered" // Common rule labels labelService = "service" @@ -124,6 +124,10 @@ func MatchesTracePipelineRule(labelSet map[string]string, unprefixedRuleName str } func matchesRule(labelSet map[string]string, unprefixedRuleName string, pipelineName string, t pipelineType) bool { + ruleName, hasRuleName := labelSet[model.AlertStatus] + if !hasRuleName { + return false + } if !matchesRuleName(labelSet, unprefixedRuleName, t) { return false } diff --git a/internal/selfmonitor/prober/log_pipeline_prober.go b/internal/selfmonitor/prober/log_pipeline_prober.go index 8711928ab..badeb5791 100644 --- a/internal/selfmonitor/prober/log_pipeline_prober.go +++ b/internal/selfmonitor/prober/log_pipeline_prober.go @@ -64,9 +64,7 @@ func (p *LogPipelineProber) someDataDropped(alerts []promv1.Alert, pipelineName } func (p *LogPipelineProber) noLogsDelivered(alerts []promv1.Alert, pipelineName string) bool { - receiverReadLogs := p.isFiring(alerts, config.RuleNameLogAgentReceiverReadLogs, pipelineName) - exporterSentLogs := p.isFiring(alerts, config.RuleNameLogAgentExporterSentLogs, pipelineName) - return receiverReadLogs && !exporterSentLogs + return p.isFiring(alerts, config.RuleNameLogAgentNoLogsDelivered, pipelineName) } func (p *LogPipelineProber) bufferFillingUp(alerts []promv1.Alert, pipelineName string) bool { @@ -78,11 +76,8 @@ func (p *LogPipelineProber) healthy(alerts []promv1.Alert, pipelineName string) bufferInUse := p.isFiring(alerts, config.RuleNameLogAgentBufferInUse, pipelineName) bufferFull := p.isFiring(alerts, config.RuleNameLogAgentBufferFull, pipelineName) exporterDroppedLogs := p.isFiring(alerts, config.RuleNameLogAgentExporterDroppedLogs, pipelineName) - - // The pipeline is healthy if either no logs are being read or all logs are being sent - receiverReadLogs := p.isFiring(alerts, config.RuleNameLogAgentReceiverReadLogs, pipelineName) - exporterSentLogs := p.isFiring(alerts, config.RuleNameLogAgentExporterSentLogs, pipelineName) - return !(bufferInUse || bufferFull || exporterDroppedLogs) && (!receiverReadLogs || exporterSentLogs) + noLogsDelivered := !p.isFiring(alerts, config.RuleNameLogAgentNoLogsDelivered, pipelineName) + return !(bufferInUse || bufferFull || exporterDroppedLogs || noLogsDelivered) } func (p *LogPipelineProber) isFiring(alerts []promv1.Alert, ruleName, pipelineName string) bool { From 0d5b71965bb10c4ae0489c65b4464868c22ab935 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Tue, 27 Aug 2024 17:37:57 +0200 Subject: [PATCH 2/9] Reduce interval --- config/manager/kustomization.yaml | 4 ++-- config/samples/telemetry_v1alpha1_logpipeline.yaml | 4 ++-- internal/selfmonitor/config/fluent_bit_rule_builder.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index a56c2e687..4503367ed 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -7,5 +7,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: europe-docker.pkg.dev/kyma-project/prod/telemetry-manager - newTag: main + newName: skhalashatsap/telemetry-operator + newTag: latest diff --git a/config/samples/telemetry_v1alpha1_logpipeline.yaml b/config/samples/telemetry_v1alpha1_logpipeline.yaml index aee81e82a..cbdc813f4 100644 --- a/config/samples/telemetry_v1alpha1_logpipeline.yaml +++ b/config/samples/telemetry_v1alpha1_logpipeline.yaml @@ -10,7 +10,7 @@ metadata: name: log-sink namespace: log-sink spec: - replicas: 1 + replicas: 0 selector: matchLabels: app: log-sink @@ -76,7 +76,7 @@ spec: - image: mingrammer/flog:latest name: flog args: - - -b=64 + - -b=1 - -f=json - -l --- diff --git a/internal/selfmonitor/config/fluent_bit_rule_builder.go b/internal/selfmonitor/config/fluent_bit_rule_builder.go index c87998303..35e6b82ac 100644 --- a/internal/selfmonitor/config/fluent_bit_rule_builder.go +++ b/internal/selfmonitor/config/fluent_bit_rule_builder.go @@ -86,7 +86,7 @@ func (rb fluentBitRuleBuilder) noLogsDeliveredRule() Rule { %s and %s`, exporterNotSentExpr, receiverReadExpr), - For: 3 * time.Minute, + For: 1 * time.Minute, } } From 5dbd45e803983cb92b74e8915f65f64b6d922cd8 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 10:37:34 +0200 Subject: [PATCH 3/9] Fix --- internal/selfmonitor/config/rules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/selfmonitor/config/rules.go b/internal/selfmonitor/config/rules.go index 6e8597e1d..44a52723e 100644 --- a/internal/selfmonitor/config/rules.go +++ b/internal/selfmonitor/config/rules.go @@ -124,7 +124,7 @@ func MatchesTracePipelineRule(labelSet map[string]string, unprefixedRuleName str } func matchesRule(labelSet map[string]string, unprefixedRuleName string, pipelineName string, t pipelineType) bool { - ruleName, hasRuleName := labelSet[model.AlertStatus] + ruleName, hasRuleName := labelSet[model.AlertNameLabel] if !hasRuleName { return false } From 2f5b2ee04a5c0007f333f5a6f972bf7722343945 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 10:43:28 +0200 Subject: [PATCH 4/9] Fix --- internal/selfmonitor/config/rules.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/selfmonitor/config/rules.go b/internal/selfmonitor/config/rules.go index 44a52723e..d0ef7739a 100644 --- a/internal/selfmonitor/config/rules.go +++ b/internal/selfmonitor/config/rules.go @@ -17,6 +17,7 @@ const ( // Fluent Bit rule names. Note that the actual full names will be prefixed with Log RuleNameLogAgentExporterSentLogs = "AgentExporterSentLogs" + RuleNameLogAgentReceiverReadLogs = "AgentReceiverReadLogs" RuleNameLogAgentExporterDroppedLogs = "AgentExporterDroppedLogs" RuleNameLogAgentBufferInUse = "AgentBufferInUse" RuleNameLogAgentBufferFull = "AgentBufferFull" @@ -124,10 +125,6 @@ func MatchesTracePipelineRule(labelSet map[string]string, unprefixedRuleName str } func matchesRule(labelSet map[string]string, unprefixedRuleName string, pipelineName string, t pipelineType) bool { - ruleName, hasRuleName := labelSet[model.AlertNameLabel] - if !hasRuleName { - return false - } if !matchesRuleName(labelSet, unprefixedRuleName, t) { return false } From ebf11bd852f6d74e05b64c361124066b9c2c4695 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 14:15:31 +0200 Subject: [PATCH 5/9] Add rule test --- internal/selfmonitor/config/expr_builder.go | 12 ++++++++++++ .../config/fluent_bit_rule_builder.go | 19 +++++++++---------- internal/selfmonitor/config/rules_test.go | 16 ++++++++-------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/internal/selfmonitor/config/expr_builder.go b/internal/selfmonitor/config/expr_builder.go index 93960a798..0e1170017 100644 --- a/internal/selfmonitor/config/expr_builder.go +++ b/internal/selfmonitor/config/expr_builder.go @@ -86,3 +86,15 @@ func (eb *exprBuilder) equal(value float64) *exprBuilder { func (eb *exprBuilder) build() string { return eb.expr } + +func and(exprs ...string) string { + return strings.Join(wrapInParentheses(exprs), " and ") +} + +func wrapInParentheses(input []string) []string { + wrapped := make([]string, len(input)) + for i, str := range input { + wrapped[i] = fmt.Sprintf("(%s)", str) + } + return wrapped +} diff --git a/internal/selfmonitor/config/fluent_bit_rule_builder.go b/internal/selfmonitor/config/fluent_bit_rule_builder.go index 35e6b82ac..87eca16e6 100644 --- a/internal/selfmonitor/config/fluent_bit_rule_builder.go +++ b/internal/selfmonitor/config/fluent_bit_rule_builder.go @@ -1,7 +1,6 @@ package config import ( - "fmt" "time" ) @@ -16,6 +15,9 @@ const ( bufferUsage300MB = 300000000 bufferUsage900MB = 900000000 + + // alertWaitTime is the time the alert have a pending state before firing + alertWaitTime = 1 * time.Minute ) type fluentBitRuleBuilder struct { @@ -70,23 +72,20 @@ func (rb fluentBitRuleBuilder) bufferFullRule() Rule { } func (rb fluentBitRuleBuilder) noLogsDeliveredRule() Rule { - exporterNotSentExpr := rate(metricFluentBitOutputProcBytesTotal, selectService(fluentBitMetricsServiceName)). + receiverReadExpr := rate(metricFluentBitInputBytesTotal, selectService(fluentBitMetricsServiceName)). sumBy(labelPipelineName). - equal(0). + greaterThan(0). build() - receiverReadExpr := rate(metricFluentBitInputBytesTotal, selectService(fluentBitMetricsServiceName)). + exporterNotSentExpr := rate(metricFluentBitOutputProcBytesTotal, selectService(fluentBitMetricsServiceName)). sumBy(labelPipelineName). - greaterThan(0). + equal(0). build() return Rule{ Alert: rb.namePrefix() + RuleNameLogAgentNoLogsDelivered, - Expr: fmt.Sprintf(` -%s -and -%s`, exporterNotSentExpr, receiverReadExpr), - For: 1 * time.Minute, + Expr: and(receiverReadExpr, exporterNotSentExpr), + For: alertWaitTime, } } diff --git a/internal/selfmonitor/config/rules_test.go b/internal/selfmonitor/config/rules_test.go index 3523f181a..f09f7d4cc 100644 --- a/internal/selfmonitor/config/rules_test.go +++ b/internal/selfmonitor/config/rules_test.go @@ -48,17 +48,17 @@ func TestMakeRules(t *testing.T) { require.Equal(t, "LogAgentExporterSentLogs", ruleGroup.Rules[10].Alert) require.Equal(t, "sum by (pipeline_name) (rate(fluentbit_output_proc_bytes_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) > 0", ruleGroup.Rules[10].Expr) - require.Equal(t, "LogAgentReceiverReadLogs", ruleGroup.Rules[11].Alert) - require.Equal(t, "sum by (pipeline_name) (rate(fluentbit_input_bytes_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) > 0", ruleGroup.Rules[11].Expr) + require.Equal(t, "LogAgentExporterDroppedLogs", ruleGroup.Rules[11].Alert) + require.Equal(t, "sum by (pipeline_name) (rate(fluentbit_output_dropped_records_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) > 0", ruleGroup.Rules[11].Expr) - require.Equal(t, "LogAgentExporterDroppedLogs", ruleGroup.Rules[12].Alert) - require.Equal(t, "sum by (pipeline_name) (rate(fluentbit_output_dropped_records_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) > 0", ruleGroup.Rules[12].Expr) + require.Equal(t, "LogAgentBufferInUse", ruleGroup.Rules[12].Alert) + require.Equal(t, "telemetry_fsbuffer_usage_bytes{service=\"telemetry-fluent-bit-exporter-metrics\"} > 300000000", ruleGroup.Rules[12].Expr) - require.Equal(t, "LogAgentBufferInUse", ruleGroup.Rules[13].Alert) - require.Equal(t, "telemetry_fsbuffer_usage_bytes{service=\"telemetry-fluent-bit-exporter-metrics\"} > 300000000", ruleGroup.Rules[13].Expr) + require.Equal(t, "LogAgentBufferFull", ruleGroup.Rules[13].Alert) + require.Equal(t, "telemetry_fsbuffer_usage_bytes{service=\"telemetry-fluent-bit-exporter-metrics\"} > 900000000", ruleGroup.Rules[13].Expr) - require.Equal(t, "LogAgentBufferFull", ruleGroup.Rules[14].Alert) - require.Equal(t, "telemetry_fsbuffer_usage_bytes{service=\"telemetry-fluent-bit-exporter-metrics\"} > 900000000", ruleGroup.Rules[14].Expr) + require.Equal(t, "LogAgentNoLogsDelivered", ruleGroup.Rules[14].Alert) + require.Equal(t, "(sum by (pipeline_name) (rate(fluentbit_input_bytes_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) > 0) and (sum by (pipeline_name) (rate(fluentbit_output_proc_bytes_total{service=\"telemetry-fluent-bit-metrics\"}[5m])) == 0)", ruleGroup.Rules[14].Expr) } func TestMatchesLogPipelineRule(t *testing.T) { From 65106a6890c0093d3ed72eda7c7a25c02cad8151 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 14:40:28 +0200 Subject: [PATCH 6/9] Fix --- internal/selfmonitor/prober/log_pipeline_prober.go | 2 +- .../selfmonitor/prober/log_pipeline_prober_test.go | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/internal/selfmonitor/prober/log_pipeline_prober.go b/internal/selfmonitor/prober/log_pipeline_prober.go index badeb5791..ff5494837 100644 --- a/internal/selfmonitor/prober/log_pipeline_prober.go +++ b/internal/selfmonitor/prober/log_pipeline_prober.go @@ -76,7 +76,7 @@ func (p *LogPipelineProber) healthy(alerts []promv1.Alert, pipelineName string) bufferInUse := p.isFiring(alerts, config.RuleNameLogAgentBufferInUse, pipelineName) bufferFull := p.isFiring(alerts, config.RuleNameLogAgentBufferFull, pipelineName) exporterDroppedLogs := p.isFiring(alerts, config.RuleNameLogAgentExporterDroppedLogs, pipelineName) - noLogsDelivered := !p.isFiring(alerts, config.RuleNameLogAgentNoLogsDelivered, pipelineName) + noLogsDelivered := p.isFiring(alerts, config.RuleNameLogAgentNoLogsDelivered, pipelineName) return !(bufferInUse || bufferFull || exporterDroppedLogs || noLogsDelivered) } diff --git a/internal/selfmonitor/prober/log_pipeline_prober_test.go b/internal/selfmonitor/prober/log_pipeline_prober_test.go index 7d1e42c36..d6d18a312 100644 --- a/internal/selfmonitor/prober/log_pipeline_prober_test.go +++ b/internal/selfmonitor/prober/log_pipeline_prober_test.go @@ -222,13 +222,13 @@ func TestLogPipelineProber(t *testing.T) { }, }, { - name: "receiver read logs and exporter did not send logs firing", + name: "no logs delivered firing", pipelineName: "cls", alerts: promv1.AlertsResult{ Alerts: []promv1.Alert{ { Labels: model.LabelSet{ - "alertname": "LogAgentReceiverReadLogs", + "alertname": "LogAgentNoLogsDelivered", "pipeline_name": "cls", }, State: promv1.AlertStateFiring, @@ -240,17 +240,10 @@ func TestLogPipelineProber(t *testing.T) { }, }, { - name: "exporter read logs and exporter sent logs firing", + name: "exporter sent logs firing", pipelineName: "cls", alerts: promv1.AlertsResult{ Alerts: []promv1.Alert{ - { - Labels: model.LabelSet{ - "alertname": "LogAgentReceiverReadLogs", - "pipeline_name": "cls", - }, - State: promv1.AlertStateFiring, - }, { Labels: model.LabelSet{ "alertname": "LogAgentExporterSentLogs", From 93f507c6eada4fe2dfabf249f71bc96df4fe8f3c Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 14:44:44 +0200 Subject: [PATCH 7/9] Revert leftover --- config/manager/kustomization.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 4503367ed..a56c2e687 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -7,5 +7,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: skhalashatsap/telemetry-operator - newTag: latest + newName: europe-docker.pkg.dev/kyma-project/prod/telemetry-manager + newTag: main From a5821fc657a33d4e871d5b62d5a2f5cf5941f7d9 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 14:45:45 +0200 Subject: [PATCH 8/9] Revert sample --- config/samples/telemetry_v1alpha1_logpipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/telemetry_v1alpha1_logpipeline.yaml b/config/samples/telemetry_v1alpha1_logpipeline.yaml index cbdc813f4..aee81e82a 100644 --- a/config/samples/telemetry_v1alpha1_logpipeline.yaml +++ b/config/samples/telemetry_v1alpha1_logpipeline.yaml @@ -10,7 +10,7 @@ metadata: name: log-sink namespace: log-sink spec: - replicas: 0 + replicas: 1 selector: matchLabels: app: log-sink @@ -76,7 +76,7 @@ spec: - image: mingrammer/flog:latest name: flog args: - - -b=1 + - -b=64 - -f=json - -l --- From 73e5510c46dfb4173f77538f508e39d3d918bf57 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 28 Aug 2024 15:04:13 +0200 Subject: [PATCH 9/9] Add tests --- .../prober/log_pipeline_prober_test.go | 19 +++++++++++++++++++ .../prober/otel_pipeline_prober_test.go | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/internal/selfmonitor/prober/log_pipeline_prober_test.go b/internal/selfmonitor/prober/log_pipeline_prober_test.go index d6d18a312..9b133eb68 100644 --- a/internal/selfmonitor/prober/log_pipeline_prober_test.go +++ b/internal/selfmonitor/prober/log_pipeline_prober_test.go @@ -61,6 +61,25 @@ func TestLogPipelineProber(t *testing.T) { }, }, }, + { + name: "pending alert", + pipelineName: "cls", + alerts: promv1.AlertsResult{ + Alerts: []promv1.Alert{ + { + Labels: model.LabelSet{ + "alertname": "LogAgentBufferFull", + }, + State: promv1.AlertStatePending, + }, + }, + }, + expected: LogPipelineProbeResult{ + PipelineProbeResult: PipelineProbeResult{ + Healthy: true, + }, + }, + }, { name: "alert missing pipeline_name label", pipelineName: "cls", diff --git a/internal/selfmonitor/prober/otel_pipeline_prober_test.go b/internal/selfmonitor/prober/otel_pipeline_prober_test.go index 96175cc9b..00e96822e 100644 --- a/internal/selfmonitor/prober/otel_pipeline_prober_test.go +++ b/internal/selfmonitor/prober/otel_pipeline_prober_test.go @@ -61,6 +61,25 @@ func TestOTelPipelineProber(t *testing.T) { }, }, }, + { + name: "pending alert", + pipelineName: "cls", + alerts: promv1.AlertsResult{ + Alerts: []promv1.Alert{ + { + Labels: model.LabelSet{ + "alertname": "TraceGatewayExporterDroppedData", + }, + State: promv1.AlertStatePending, + }, + }, + }, + expected: OTelPipelineProbeResult{ + PipelineProbeResult: PipelineProbeResult{ + Healthy: true, + }, + }, + }, { name: "alert missing pipeline_name label", pipelineName: "cls",