From 89cb7f1464ff7738277ecc344aaa67086a8e108f Mon Sep 17 00:00:00 2001 From: Robert Winkler Date: Fri, 16 Aug 2019 14:49:32 +0200 Subject: [PATCH] =?UTF-8?q?Issue=20#547:=20Added=20slow=20call=20rate=20to?= =?UTF-8?q?=20CircuitBreakerMetricsCollector=20an=E2=80=A6=20(#574)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Issue #547: Added slow call rate to CircuitBreakerMetricsCollector and TaggedCircuitBreakerMetrics. --- .../FixedSizeSlidingWindowMetrics.java | 6 ++--- .../metrics/SlidingTimeWindowMetrics.java | 2 +- .../tagged/TaggedCircuitBreakerMetrics.java | 23 +++++++++++++++++++ .../TaggedCircuitBreakerMetricsTest.java | 14 ++++++----- .../CircuitBreakerMetricsCollector.java | 22 +++++++++++++++++- .../CircuitBreakerMetricsCollectorTest.java | 15 ++++++++++-- .../ratpack/Resilience4jModuleSpec.groovy | 1 + 7 files changed, 70 insertions(+), 13 deletions(-) diff --git a/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/FixedSizeSlidingWindowMetrics.java b/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/FixedSizeSlidingWindowMetrics.java index 629d2d8d8b..9c3957cd4e 100644 --- a/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/FixedSizeSlidingWindowMetrics.java +++ b/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/FixedSizeSlidingWindowMetrics.java @@ -28,10 +28,10 @@ * If the time window size is 10, the circular array has always 10 measurements. * * The sliding window incrementally updates a total aggregation. - * The total aggregation is updated incrementally when a new call outcome is recorded. When the oldest bucket is evicted, the partial totalAggregation of that bucket - * is subtracted from the total totalAggregation and the bucket is reset. (Subtract-on-Evict) + * The total aggregation is updated incrementally when a new call outcome is recorded. When the oldest measurement is evicted, the measurement + * is subtracted from the total aggregation. (Subtract-on-Evict) * - * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the time window size. + * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the window size. * The space requirement (memory consumption) of this implementation should be O(n). */ public class FixedSizeSlidingWindowMetrics implements Metrics { diff --git a/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/SlidingTimeWindowMetrics.java b/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/SlidingTimeWindowMetrics.java index 05d068d6f5..99949529b0 100644 --- a/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/SlidingTimeWindowMetrics.java +++ b/resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/SlidingTimeWindowMetrics.java @@ -34,7 +34,7 @@ * * The sliding window does not store call outcomes (tuples) individually, but incrementally updates partial aggregations (bucket) and a total aggregation. * The total total aggregation is updated incrementally when a new call outcome is recorded. When the oldest bucket is evicted, the partial total aggregation of that bucket - * is subtracted from the total totalAggregation and the bucket is reset. (Subtract-on-Evict) + * is subtracted from the total aggregation. (Subtract-on-Evict) * * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the time window size. * The space requirement (memory consumption) of this implementation should be nearly constant O(n), since the call outcomes (tuples) are not stored individually. diff --git a/resilience4j-micrometer/src/main/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetrics.java b/resilience4j-micrometer/src/main/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetrics.java index 2be2a6b9f8..c52ad0579f 100644 --- a/resilience4j-micrometer/src/main/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetrics.java +++ b/resilience4j-micrometer/src/main/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetrics.java @@ -93,6 +93,11 @@ private void addMetrics(MeterRegistry registry, CircuitBreaker circuitBreaker) { .tag(TagNames.NAME, circuitBreaker.getName()) .register(registry).getId()); + idSet.add(Gauge.builder(names.getSlowCallRateMetricName(), circuitBreaker, cb -> cb.getMetrics().getSlowCallRate()) + .description("The slow call of the circuit breaker") + .tag(TagNames.NAME, circuitBreaker.getName()) + .register(registry).getId()); + Timer successfulCalls = Timer.builder(names.getCallsMetricName()) .description("Total number of successful calls") .tag(TagNames.NAME, circuitBreaker.getName()) @@ -153,6 +158,7 @@ public static class MetricNames { public static final String DEFAULT_CIRCUIT_BREAKER_STATE = DEFAULT_PREFIX + ".state"; public static final String DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS = DEFAULT_PREFIX + ".buffered.calls"; public static final String DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE = DEFAULT_PREFIX + ".failure.rate"; + public static final String DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE = DEFAULT_PREFIX + ".slow.call.rate"; /** * Returns a builder for creating custom metric names. @@ -174,6 +180,7 @@ public static MetricNames ofDefaults() { private String stateMetricName = DEFAULT_CIRCUIT_BREAKER_STATE; private String bufferedCallsMetricName = DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS; private String failureRateMetricName = DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE; + private String slowCallRateMetricName = DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE; private MetricNames() {} @@ -205,6 +212,13 @@ public String getFailureRateMetricName() { return failureRateMetricName; } + /** Returns the metric name for slow call rate, defaults to {@value DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE}. + * @return The failure rate metric name. + */ + public String getSlowCallRateMetricName() { + return slowCallRateMetricName; + } + /** Helps building custom instance of {@link MetricNames}. */ public static class Builder { private final MetricNames metricNames = new MetricNames(); @@ -244,6 +258,15 @@ public Builder failureRateMetricName(String failureRateMetricName) { return this; } + /** Overrides the default metric name {@value MetricNames#DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE} with a given one. + * @param slowCallRateMetricName The slow call rate metric name. + * @return The builder. + */ + public Builder slowCallRateMetricName(String slowCallRateMetricName) { + metricNames.slowCallRateMetricName = requireNonNull(slowCallRateMetricName); + return this; + } + /** Builds {@link MetricNames} instance. * @return The built {@link MetricNames} instance. */ diff --git a/resilience4j-micrometer/src/test/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetricsTest.java b/resilience4j-micrometer/src/test/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetricsTest.java index bcada61411..618ae886e3 100644 --- a/resilience4j-micrometer/src/test/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetricsTest.java +++ b/resilience4j-micrometer/src/test/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetricsTest.java @@ -61,11 +61,11 @@ public void shouldAddMetricsForANewlyCreatedCircuitBreaker() { newCircuitBreaker.onSuccess(0, TimeUnit.NANOSECONDS); assertThat(taggedCircuitBreakerMetrics.meterIdMap).containsKeys("backendA", "backendB"); - assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendA")).hasSize(12); - assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendB")).hasSize(12); + assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendA")).hasSize(13); + assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendB")).hasSize(13); List meters = meterRegistry.getMeters(); - assertThat(meters).hasSize(24); + assertThat(meters).hasSize(26); Collection gauges = meterRegistry.get(DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS).gauges(); @@ -77,7 +77,7 @@ public void shouldAddMetricsForANewlyCreatedCircuitBreaker() { @Test public void shouldRemovedMetricsForRemovedRetry() { List meters = meterRegistry.getMeters(); - assertThat(meters).hasSize(12); + assertThat(meters).hasSize(13); assertThat(taggedCircuitBreakerMetrics.meterIdMap).containsKeys("backendA"); circuitBreakerRegistry.remove("backendA"); @@ -91,7 +91,7 @@ public void shouldRemovedMetricsForRemovedRetry() { @Test public void notPermittedCallsCounterReportsCorrespondingValue() { List meters = meterRegistry.getMeters(); - assertThat(meters).hasSize(12); + assertThat(meters).hasSize(13); Collection counters = meterRegistry.get(DEFAULT_CIRCUIT_BREAKER_CALLS).counters(); @@ -146,6 +146,7 @@ public void metricsAreRegisteredWithCustomName() { .stateMetricName("custom_state") .bufferedCallsMetricName("custom_buffered_calls") .failureRateMetricName("custom_failure_rate") + .slowCallRateMetricName("custom_slow_call_rate") .build(), circuitBreakerRegistry ).bindTo(meterRegistry); @@ -160,7 +161,8 @@ public void metricsAreRegisteredWithCustomName() { "custom_calls", "custom_state", "custom_buffered_calls", - "custom_failure_rate" + "custom_failure_rate", + "custom_slow_call_rate" )); } diff --git a/resilience4j-prometheus/src/main/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollector.java b/resilience4j-prometheus/src/main/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollector.java index 9eb5bd5eea..b00a6b30eb 100644 --- a/resilience4j-prometheus/src/main/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollector.java +++ b/resilience4j-prometheus/src/main/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollector.java @@ -112,6 +112,12 @@ private List collectGaugeSamples() { LabelNames.NAME ); + GaugeMetricFamily slowCallRateFamily = new GaugeMetricFamily( + names.getSlowCallRateMetricName(), + "The slow call rate", + LabelNames.NAME + ); + for (CircuitBreaker circuitBreaker : this.circuitBreakerRegistry.getAllCircuitBreakers()) { final CircuitBreaker.State[] states = CircuitBreaker.State.values(); for (CircuitBreaker.State state : states) { @@ -124,8 +130,9 @@ private List collectGaugeSamples() { bufferedCallsFamily.addMetric(asList(circuitBreaker.getName(), KIND_SUCCESSFUL), metrics.getNumberOfSuccessfulCalls()); bufferedCallsFamily.addMetric(asList(circuitBreaker.getName(), KIND_FAILED), metrics.getNumberOfFailedCalls()); failureRateFamily.addMetric(nameLabel, metrics.getFailureRate()); + slowCallRateFamily.addMetric(nameLabel, metrics.getSlowCallRate()); } - return asList(stateFamily, bufferedCallsFamily, failureRateFamily); + return asList(stateFamily, bufferedCallsFamily, failureRateFamily, slowCallRateFamily); } /** Defines possible configuration for metric names. */ @@ -135,6 +142,7 @@ public static class MetricNames { public static final String DEFAULT_CIRCUIT_BREAKER_STATE = "resilience4j_circuitbreaker_state"; public static final String DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS = "resilience4j_circuitbreaker_buffered_calls"; public static final String DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE = "resilience4j_circuitbreaker_failure_rate"; + public static final String DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE = "resilience4j_circuitbreaker_slow_call_rate"; /** * Returns a builder for creating custom metric names. @@ -153,6 +161,7 @@ public static MetricNames ofDefaults() { private String stateMetricName = DEFAULT_CIRCUIT_BREAKER_STATE; private String bufferedCallsMetricName = DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS; private String failureRateMetricName = DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE; + private String slowCallRateMetricName = DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE; private MetricNames() {} @@ -171,6 +180,11 @@ public String getFailureRateMetricName() { return failureRateMetricName; } + /** Returns the metric name for slow call rate, defaults to {@value DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE}. */ + public String getSlowCallRateMetricName() { + return slowCallRateMetricName; + } + /** Returns the metric name for state, defaults to {@value DEFAULT_CIRCUIT_BREAKER_STATE}. */ public String getStateMetricName() { return stateMetricName; @@ -204,6 +218,12 @@ public Builder failureRateMetricName(String failureRateMetricName) { return this; } + /** Overrides the default metric name {@value MetricNames#DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE} with a given one. */ + public Builder slowCallRateMetricName(String slowCallRateMetricName) { + metricNames.slowCallRateMetricName = requireNonNull(slowCallRateMetricName); + return this; + } + /** Builds {@link MetricNames} instance. */ public MetricNames build() { return metricNames; diff --git a/resilience4j-prometheus/src/test/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollectorTest.java b/resilience4j-prometheus/src/test/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollectorTest.java index 8d2e438a3e..3005c007c9 100644 --- a/resilience4j-prometheus/src/test/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollectorTest.java +++ b/resilience4j-prometheus/src/test/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollectorTest.java @@ -41,8 +41,8 @@ public void setup() { CircuitBreakerMetricsCollector.ofCircuitBreakerRegistry(circuitBreakerRegistry).register(registry); // record some basic stats - circuitBreaker.onSuccess(0, TimeUnit.NANOSECONDS); - circuitBreaker.onError(0, TimeUnit.NANOSECONDS, new RuntimeException("oops")); + circuitBreaker.onSuccess(100, TimeUnit.NANOSECONDS); + circuitBreaker.onError(100, TimeUnit.NANOSECONDS, new RuntimeException("oops")); circuitBreaker.transitionToOpenState(); } @@ -163,6 +163,17 @@ public void failureRateReportsCorrespondingValue() { assertThat(failureRate.floatValue()).isEqualTo(circuitBreaker.getMetrics().getFailureRate()); } + @Test + public void slowCallRateReportsCorrespondingValue() { + Double slowCallRate = registry.getSampleValue( + DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE, + new String[]{"name"}, + new String[]{circuitBreaker.getName()} + ); + + assertThat(slowCallRate.floatValue()).isEqualTo(circuitBreaker.getMetrics().getSlowCallRate()); + } + @Test public void customMetricNamesOverrideDefaultOnes() { CollectorRegistry registry = new CollectorRegistry(); diff --git a/resilience4j-ratpack/src/test/groovy/io/github/resilience4j/ratpack/Resilience4jModuleSpec.groovy b/resilience4j-ratpack/src/test/groovy/io/github/resilience4j/ratpack/Resilience4jModuleSpec.groovy index 9a63bc61a9..e90a7bb3be 100644 --- a/resilience4j-ratpack/src/test/groovy/io/github/resilience4j/ratpack/Resilience4jModuleSpec.groovy +++ b/resilience4j-ratpack/src/test/groovy/io/github/resilience4j/ratpack/Resilience4jModuleSpec.groovy @@ -820,6 +820,7 @@ class Resilience4jModuleSpec extends Specification { then: families == ['resilience4j_bulkhead_available_concurrent_calls', 'resilience4j_bulkhead_max_allowed_concurrent_calls', + 'resilience4j_circuitbreaker_slow_call_rate', 'resilience4j_circuitbreaker_buffered_calls', 'resilience4j_circuitbreaker_calls', 'resilience4j_circuitbreaker_state',