diff --git a/src/prometheus/grpc_alerts.rs b/src/prometheus/grpc_alerts.rs index 6e211d7..70a3645 100644 --- a/src/prometheus/grpc_alerts.rs +++ b/src/prometheus/grpc_alerts.rs @@ -38,24 +38,47 @@ fn grpc_promql( ) -> String { match network_alert { NetworkAlert::ErrorPercent => { - todo!() - } - NetworkAlert::TrafficPerSecond => { - todo!() + format!("(sum by (destination_workload) (rate(grpc_server_handled_total{{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\", destination_workload=\"{0}\"}}[{1}]))/ sum by (destination_workload) (rate(grpc_server_started_total{{destination_workload=\"{0}\"}}[{1}])) * 100 > {2})", + _spec.deployment_name, + alert_config.for_, + alert_config.value + ) } + NetworkAlert::TrafficPerSecond => "this is a placeholder value.".to_string(), NetworkAlert::LatencyMillisecondsP50 => { - todo!() + format!( + "histogram_quantile(0.50, istio_request_duration_milliseconds{{destination_workload={0}}}[{1}]) {2} {3}", + _spec.deployment_name, + alert_config.for_, + alert_config.operation, + alert_config.value + ) } NetworkAlert::LatencyMillisecondsP90 => { - todo!() + format!( + "histogram_quantile(0.90, istio_request_duration_milliseconds{{destination_workload={0}}}[{1}]) {2} {3}", + _spec.deployment_name, + alert_config.for_, + alert_config.operation, + alert_config.value + ) } NetworkAlert::LatencyMillisecondsP95 => { - todo!() + format!( + "histogram_quantile(0.95, istio_request_duration_milliseconds{{destination_workload={0}}}[{1}]) {2} {3}", + _spec.deployment_name, + alert_config.for_, + alert_config.operation, + alert_config.value + ) } NetworkAlert::LatencyMillisecondsP99 => { format!( - "histogram_quantile(0.99, istio_request_duration_milliseconds{}[{0}])", - alert_config.for_ + "histogram_quantile(0.99, istio_request_duration_milliseconds{{destination_workload={0}}}[{1}]) {2} {3}", + _spec.deployment_name, + alert_config.for_, + alert_config.operation, + alert_config.value ) } } diff --git a/src/prometheus/http_alerts.rs b/src/prometheus/http_alerts.rs index 406898e..3b3d729 100644 --- a/src/prometheus/http_alerts.rs +++ b/src/prometheus/http_alerts.rs @@ -78,7 +78,7 @@ fn error_percent_annotations(alert_config: &AlertConfig) -> Annotations { fn latency_percentile_alerts( spec: &ServiceAlertSpec, - percentile: i8, + _percentile: i8, alert_configs: &[AlertConfig], ) -> Vec { alert_configs @@ -86,7 +86,13 @@ fn latency_percentile_alerts( .enumerate() .map(|(i, conf)| AlertRules { alert: format!("HTTPLatencyPercentileRule-{0}-{1}", spec.deployment_name, i), - expr: format!(r#"latency percentile {} {}"#, conf.operation, percentile), + expr: format!( + "histogram_quantile({3}, istio_requests_total{{destination_workload={0}}}[{1}]) {2} {3}", + spec.deployment_name, + conf.for_, + conf.operation, + conf.value + ), for_: conf.for_.clone(), labels: Labels { severity: PrometheusSeverity::from(&conf.with_labels),