Issue ReactiveX#547: Added slow call rate to CircuitBreakerMetricsCol…

…lector an… (ReactiveX#574)

* Issue ReactiveX#547: Added slow call rate to CircuitBreakerMetricsCollector and TaggedCircuitBreakerMetrics.

resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/FixedSizeSlidingWindowMetrics.java

@@ -28,10 +28,10 @@
  * If the time window size is 10, the circular array has always 10 measurements.
  *
  * The sliding window incrementally updates a total aggregation.
- * The total aggregation is updated incrementally when a new call outcome is recorded. When the oldest bucket is evicted, the partial totalAggregation of that bucket
- * is subtracted from the total totalAggregation and the bucket is reset. (Subtract-on-Evict)
+ * The total aggregation is updated incrementally when a new call outcome is recorded. When the oldest measurement is evicted, the measurement
+ * is subtracted from the total aggregation. (Subtract-on-Evict)
  *
- * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the time window size.
+ * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the window size.
  * The space requirement (memory consumption) of this implementation should be O(n).
  */
 public class FixedSizeSlidingWindowMetrics implements Metrics {

resilience4j-core/src/main/java/io/github/resilience4j/core/metrics/SlidingTimeWindowMetrics.java

@@ -34,7 +34,7 @@
  *
  * The sliding window does not store call outcomes (tuples) individually, but incrementally updates partial aggregations (bucket) and a total aggregation.
  * The total total aggregation is updated incrementally when a new call outcome is recorded. When the oldest bucket is evicted, the partial total aggregation of that bucket
- * is subtracted from the total totalAggregation and the bucket is reset. (Subtract-on-Evict)
+ * is subtracted from the total aggregation. (Subtract-on-Evict)
  *
  * The time to retrieve a Snapshot is constant 0(1), since the Snapshot is pre-aggregated and is independent of the time window size.
  * The space requirement (memory consumption) of this implementation should be nearly constant O(n), since the call outcomes (tuples) are not stored individually.

resilience4j-micrometer/src/main/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetrics.java

@@ -93,6 +93,11 @@ private void addMetrics(MeterRegistry registry, CircuitBreaker circuitBreaker) {
                 .tag(TagNames.NAME, circuitBreaker.getName())
                 .register(registry).getId());
 
+        idSet.add(Gauge.builder(names.getSlowCallRateMetricName(), circuitBreaker, cb -> cb.getMetrics().getSlowCallRate())
+                .description("The slow call of the circuit breaker")
+                .tag(TagNames.NAME, circuitBreaker.getName())
+                .register(registry).getId());
+
         Timer successfulCalls = Timer.builder(names.getCallsMetricName())
                 .description("Total number of successful calls")
                 .tag(TagNames.NAME, circuitBreaker.getName())
@@ -153,6 +158,7 @@ public static class MetricNames {
         public static final String DEFAULT_CIRCUIT_BREAKER_STATE = DEFAULT_PREFIX + ".state";
         public static final String DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS = DEFAULT_PREFIX + ".buffered.calls";
         public static final String DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE = DEFAULT_PREFIX + ".failure.rate";
+        public static final String DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE = DEFAULT_PREFIX + ".slow.call.rate";
 
         /**
          * Returns a builder for creating custom metric names.
@@ -174,6 +180,7 @@ public static MetricNames ofDefaults() {
         private String stateMetricName = DEFAULT_CIRCUIT_BREAKER_STATE;
         private String bufferedCallsMetricName = DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS;
         private String failureRateMetricName = DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE;
+        private String slowCallRateMetricName = DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE;
 
         private MetricNames() {}
 
@@ -205,6 +212,13 @@ public String getFailureRateMetricName() {
             return failureRateMetricName;
         }
 
+        /** Returns the metric name for slow call rate, defaults to {@value DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE}.
+         * @return The failure rate metric name.
+         */
+        public String getSlowCallRateMetricName() {
+            return slowCallRateMetricName;
+        }
+
         /** Helps building custom instance of {@link MetricNames}. */
         public static class Builder {
             private final MetricNames metricNames = new MetricNames();
@@ -244,6 +258,15 @@ public Builder failureRateMetricName(String failureRateMetricName) {
                 return this;
             }
 
+            /** Overrides the default metric name {@value MetricNames#DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE} with a given one.
+             * @param slowCallRateMetricName The slow call rate metric name.
+             * @return The builder.
+             */
+            public Builder slowCallRateMetricName(String slowCallRateMetricName) {
+                metricNames.slowCallRateMetricName = requireNonNull(slowCallRateMetricName);
+                return this;
+            }
+
             /** Builds {@link MetricNames} instance.
              * @return The built {@link MetricNames} instance.
              */

resilience4j-micrometer/src/test/java/io/github/resilience4j/micrometer/tagged/TaggedCircuitBreakerMetricsTest.java

@@ -61,11 +61,11 @@ public void shouldAddMetricsForANewlyCreatedCircuitBreaker() {
         newCircuitBreaker.onSuccess(0, TimeUnit.NANOSECONDS);
 
         assertThat(taggedCircuitBreakerMetrics.meterIdMap).containsKeys("backendA", "backendB");
-        assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendA")).hasSize(12);
-        assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendB")).hasSize(12);
+        assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendA")).hasSize(13);
+        assertThat(taggedCircuitBreakerMetrics.meterIdMap.get("backendB")).hasSize(13);
 
         List<Meter> meters = meterRegistry.getMeters();
-        assertThat(meters).hasSize(24);
+        assertThat(meters).hasSize(26);
 
         Collection<Gauge> gauges = meterRegistry.get(DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS).gauges();
 
@@ -77,7 +77,7 @@ public void shouldAddMetricsForANewlyCreatedCircuitBreaker() {
     @Test
     public void shouldRemovedMetricsForRemovedRetry() {
         List<Meter> meters = meterRegistry.getMeters();
-        assertThat(meters).hasSize(12);
+        assertThat(meters).hasSize(13);
 
         assertThat(taggedCircuitBreakerMetrics.meterIdMap).containsKeys("backendA");
         circuitBreakerRegistry.remove("backendA");
@@ -91,7 +91,7 @@ public void shouldRemovedMetricsForRemovedRetry() {
     @Test
     public void notPermittedCallsCounterReportsCorrespondingValue() {
         List<Meter> meters = meterRegistry.getMeters();
-        assertThat(meters).hasSize(12);
+        assertThat(meters).hasSize(13);
 
         Collection<Counter> counters = meterRegistry.get(DEFAULT_CIRCUIT_BREAKER_CALLS).counters();
 
@@ -146,6 +146,7 @@ public void metricsAreRegisteredWithCustomName() {
                         .stateMetricName("custom_state")
                         .bufferedCallsMetricName("custom_buffered_calls")
                         .failureRateMetricName("custom_failure_rate")
+                        .slowCallRateMetricName("custom_slow_call_rate")
                         .build(),
                 circuitBreakerRegistry
         ).bindTo(meterRegistry);
@@ -160,7 +161,8 @@ public void metricsAreRegisteredWithCustomName() {
                 "custom_calls",
                 "custom_state",
                 "custom_buffered_calls",
-                "custom_failure_rate"
+                "custom_failure_rate",
+                "custom_slow_call_rate"
         ));
     }
 

resilience4j-prometheus/src/main/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollector.java

@@ -112,6 +112,12 @@ private List<MetricFamilySamples> collectGaugeSamples() {
                 LabelNames.NAME
         );
 
+        GaugeMetricFamily slowCallRateFamily = new GaugeMetricFamily(
+                names.getSlowCallRateMetricName(),
+                "The slow call rate",
+                LabelNames.NAME
+        );
+
         for (CircuitBreaker circuitBreaker : this.circuitBreakerRegistry.getAllCircuitBreakers()) {
             final CircuitBreaker.State[] states = CircuitBreaker.State.values();
             for (CircuitBreaker.State state : states) {
@@ -124,8 +130,9 @@ private List<MetricFamilySamples> collectGaugeSamples() {
             bufferedCallsFamily.addMetric(asList(circuitBreaker.getName(), KIND_SUCCESSFUL), metrics.getNumberOfSuccessfulCalls());
             bufferedCallsFamily.addMetric(asList(circuitBreaker.getName(), KIND_FAILED), metrics.getNumberOfFailedCalls());
             failureRateFamily.addMetric(nameLabel, metrics.getFailureRate());
+            slowCallRateFamily.addMetric(nameLabel, metrics.getSlowCallRate());
         }
-        return asList(stateFamily, bufferedCallsFamily, failureRateFamily);
+        return asList(stateFamily, bufferedCallsFamily, failureRateFamily, slowCallRateFamily);
     }
 
     /** Defines possible configuration for metric names. */
@@ -135,6 +142,7 @@ public static class MetricNames {
         public static final String DEFAULT_CIRCUIT_BREAKER_STATE = "resilience4j_circuitbreaker_state";
         public static final String DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS = "resilience4j_circuitbreaker_buffered_calls";
         public static final String DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE = "resilience4j_circuitbreaker_failure_rate";
+        public static final String DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE = "resilience4j_circuitbreaker_slow_call_rate";
 
         /**
          * Returns a builder for creating custom metric names.
@@ -153,6 +161,7 @@ public static MetricNames ofDefaults() {
         private String stateMetricName = DEFAULT_CIRCUIT_BREAKER_STATE;
         private String bufferedCallsMetricName = DEFAULT_CIRCUIT_BREAKER_BUFFERED_CALLS;
         private String failureRateMetricName = DEFAULT_CIRCUIT_BREAKER_FAILURE_RATE;
+        private String slowCallRateMetricName = DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE;
 
         private MetricNames() {}
 
@@ -171,6 +180,11 @@ public String getFailureRateMetricName() {
             return failureRateMetricName;
         }
 
+        /** Returns the metric name for slow call rate, defaults to {@value DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE}. */
+        public String getSlowCallRateMetricName() {
+            return slowCallRateMetricName;
+        }
+
         /** Returns the metric name for state, defaults to {@value DEFAULT_CIRCUIT_BREAKER_STATE}. */
         public String getStateMetricName() {
             return stateMetricName;
@@ -204,6 +218,12 @@ public Builder failureRateMetricName(String failureRateMetricName) {
                 return this;
             }
 
+            /** Overrides the default metric name {@value MetricNames#DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE} with a given one. */
+            public Builder slowCallRateMetricName(String slowCallRateMetricName) {
+                metricNames.slowCallRateMetricName = requireNonNull(slowCallRateMetricName);
+                return this;
+            }
+
             /** Builds {@link MetricNames} instance. */
             public MetricNames build() {
                 return metricNames;

resilience4j-prometheus/src/test/java/io/github/resilience4j/prometheus/collectors/CircuitBreakerMetricsCollectorTest.java

@@ -41,8 +41,8 @@ public void setup() {
         CircuitBreakerMetricsCollector.ofCircuitBreakerRegistry(circuitBreakerRegistry).register(registry);
 
         // record some basic stats
-        circuitBreaker.onSuccess(0, TimeUnit.NANOSECONDS);
-        circuitBreaker.onError(0, TimeUnit.NANOSECONDS, new RuntimeException("oops"));
+        circuitBreaker.onSuccess(100, TimeUnit.NANOSECONDS);
+        circuitBreaker.onError(100, TimeUnit.NANOSECONDS, new RuntimeException("oops"));
         circuitBreaker.transitionToOpenState();
     }
 
@@ -163,6 +163,17 @@ public void failureRateReportsCorrespondingValue() {
         assertThat(failureRate.floatValue()).isEqualTo(circuitBreaker.getMetrics().getFailureRate());
     }
 
+    @Test
+    public void slowCallRateReportsCorrespondingValue() {
+        Double slowCallRate = registry.getSampleValue(
+                DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE,
+                new String[]{"name"},
+                new String[]{circuitBreaker.getName()}
+        );
+
+        assertThat(slowCallRate.floatValue()).isEqualTo(circuitBreaker.getMetrics().getSlowCallRate());
+    }
+
     @Test
     public void customMetricNamesOverrideDefaultOnes() {
         CollectorRegistry registry = new CollectorRegistry();

resilience4j-ratpack/src/test/groovy/io/github/resilience4j/ratpack/Resilience4jModuleSpec.groovy

@@ -820,6 +820,7 @@ class Resilience4jModuleSpec extends Specification {
         then:
         families == ['resilience4j_bulkhead_available_concurrent_calls',
                      'resilience4j_bulkhead_max_allowed_concurrent_calls',
+                     'resilience4j_circuitbreaker_slow_call_rate',
                      'resilience4j_circuitbreaker_buffered_calls',
                      'resilience4j_circuitbreaker_calls',
                      'resilience4j_circuitbreaker_state',