Add exemplar support to histogram (and histogram part of timer) Prometheus output

bom/pom.xml

@@ -512,6 +512,11 @@
                 <artifactId>helidon-metrics</artifactId>
                 <version>${helidon.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.helidon.metrics</groupId>
+                <artifactId>helidon-metrics-trace-exemplar</artifactId>
+                <version>${helidon.version}</version>
+            </dependency>
 
             <!-- health checks -->
             <dependency>

metrics/metrics/src/main/java/io/helidon/metrics/DefaultExemplarService.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2021 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.helidon.metrics;
+
+import java.util.function.Supplier;
+
+class DefaultExemplarService implements ExemplarService {
+
+    @Override
+    public Supplier<String> labelSupplier() {
+        return () -> "";
+    }
+}

metrics/metrics/src/main/java/io/helidon/metrics/DisplayableLabeledSnapshot.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2021 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.helidon.metrics;
+
+import io.helidon.metrics.LabeledSample.Derived;
+import io.helidon.metrics.WeightedSnapshot.WeightedSample;
+
+/**
+ * Internal interface prescribing minimum behavior of a snapshot needed to produce output.
+ */
+interface DisplayableLabeledSnapshot {
+
+    Derived value(double quantile);
+
+    Derived median();
+
+    WeightedSample max();
+
+    WeightedSample min();
+
+    Derived mean();
+
+    Derived stdDev();
+
+    Derived sample75thPercentile();
+
+    Derived sample95thPercentile();
+
+    Derived sample98thPercentile();
+
+    Derived sample99thPercentile();
+
+    Derived sample999thPercentile();
+
+
+
+}

metrics/metrics/src/main/java/io/helidon/metrics/ExemplarService.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.helidon.metrics;
+
+import java.util.function.Supplier;
+
+/**
+ * Behavior for supporting exemplars in metrics histograms.
+ */
+public interface ExemplarService {
+
+    /**
+     * Defayult priority for an {@code ExemplarService} found by the service loader without an explicit {@code @Priority}
+     * annotation.
+     */
+    int DEFAULT_PRIORITY = 1000;
+
+    /**
+     * Returns a label using whatever current context is available.
+     *
+     * @return the label
+     */
+    Supplier<String> labelSupplier();
+}

metrics/metrics/src/main/java/io/helidon/metrics/ExemplarServiceManager.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2021 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package io.helidon.metrics;
+
+import java.util.Comparator;
+import java.util.ServiceLoader;
+import java.util.StringJoiner;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Priority;
+
+import io.helidon.common.LazyValue;
+
+/**
+ * Loads the {@link ExemplarService} instance (if any) with the most urgent priority.
+ */
+class ExemplarServiceManager {
+
+    private static final Logger LOGGER = Logger.getLogger(ExemplarServiceManager.class.getName());
+
+    private static final LazyValue<Supplier<String>> EXEMPLAR_SUPPLIER =
+            LazyValue.create(ExemplarServiceManager::chooseExemplarSupplier);
+
+    private ExemplarServiceManager() {
+    }
+
+    /**
+     * Returns the current exemplar string (e.g., trace ID).
+     *
+     * @return exemplar string provided by the highest-priority service instance
+     */
+    static String exemplar() {
+        return EXEMPLAR_SUPPLIER.get().get();
+    }
+
+    private static Supplier<String> chooseExemplarSupplier() {
+        StringJoiner joiner = LOGGER.isLoggable(Level.FINE) ? new StringJoiner(",", "[", "]") : null;
+        ExemplarService exemplarService =
+            ServiceLoader.load(ExemplarService.class)
+                .stream()
+                .peek(provider -> {
+                    if (joiner != null) {
+                        joiner.add(String.format("%s(priority %d)", provider.type().getName(), priorityValue(provider)));
+                    }
+                })
+                .min(Comparator.comparing(ExemplarServiceManager::priorityValue))
+                .map(ServiceLoader.Provider::get)
+                .orElse(new DefaultExemplarService());
+        if (joiner != null) {
+            LOGGER.log(Level.FINE, String.format("Candidate ExemplarSupports: %s, using %s", joiner.toString(),
+                    exemplarService.getClass().getName()));
+        }
+
+        return exemplarService.labelSupplier();
+    }
+
+    private static int priorityValue(ServiceLoader.Provider<ExemplarService> exemplarSupportProvider) {
+        Priority p = exemplarSupportProvider.type().getAnnotation(Priority.class);
+        return p == null ? ExemplarService.DEFAULT_PRIORITY : p.value();
+    }
+
+
+}

metrics/metrics/src/main/java/io/helidon/metrics/ExponentiallyDecayingReservoir.java

@@ -23,8 +23,6 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import org.eclipse.microprofile.metrics.Snapshot;
-
 import static java.lang.Math.exp;
 import static java.lang.Math.min;
 
@@ -91,22 +89,23 @@ public int size() {
         return (int) min(size, count.get());
     }
 
-    public void update(long value) {
-        update(value, currentTimeInSeconds());
+    public void update(long value, String label) {
+        update(value, currentTimeInSeconds(), label);
     }
 
     /**
      * Adds an old value with a fixed timestamp to the reservoir.
      *
      * @param value     the value to be added
      * @param timestamp the epoch timestamp of {@code value} in seconds
+     * @param label     the optional label associated with the sample
      */
-    public void update(long value, long timestamp) {
+    public void update(long value, long timestamp, String label) {
         rescaleIfNeeded();
         lockForRegularUsage();
         try {
             final double itemWeight = weight(timestamp - startTime);
-            final WeightedSnapshot.WeightedSample sample = new WeightedSnapshot.WeightedSample(value, itemWeight);
+            final WeightedSnapshot.WeightedSample sample = new WeightedSnapshot.WeightedSample(value, itemWeight, label);
             final double priority = itemWeight / ThreadLocalRandom.current().nextDouble();
 
             final long newCount = count.incrementAndGet();
@@ -134,7 +133,7 @@ private void rescaleIfNeeded() {
         }
     }
 
-    public Snapshot getSnapshot() {
+    public WeightedSnapshot getSnapshot() {
         rescaleIfNeeded();
         lockForRegularUsage();
         try {
@@ -184,7 +183,8 @@ private void rescale(long now, long next) {
                     for (Double key : keys) {
                         final WeightedSnapshot.WeightedSample sample = values.remove(key);
                         final WeightedSnapshot.WeightedSample newSample = new WeightedSnapshot.WeightedSample(sample.getValue(),
-                                                                                                              sample.getWeight() * scalingFactor);
+                                                                                                              sample.getWeight() * scalingFactor,
+                                                                                                              sample.label());
                         if (Double.compare(newSample.getWeight(), 0) == 0) {
                             continue;
                         }

metrics/metrics/src/main/java/io/helidon/metrics/HelidonCounter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2021 Oracle and/or its affiliates.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -66,6 +66,26 @@ public String prometheusNameWithUnits(MetricID metricID) {
         return metricName.endsWith("total") ? metricName : metricName + "_total";
     }
 
+    @Override
+    public void prometheusData(StringBuilder sb, MetricID metricID, boolean withHelpType) {
+        String nameWithUnits = prometheusNameWithUnits(metricID);
+        if (withHelpType) {
+            prometheusType(sb, nameWithUnits, metadata().getType());
+            prometheusHelp(sb, nameWithUnits);
+        }
+        sb.append(nameWithUnits)
+                .append(prometheusTags(metricID.getTags()))
+                .append(" ")
+                .append(prometheusValue());
+        if (delegate instanceof CounterImpl) {
+            CounterSample sample = ((CounterImpl) delegate).counterSample;
+            if (sample != null) {
+                sb.append(prometheusExemplar(sample.label(), sample.value(), sample.timestamp()));
+            }
+        }
+        sb.append('\n');
+    }
+
     @Override
     public String prometheusValue() {
         return Long.toString(getCount());
@@ -79,14 +99,17 @@ public void jsonData(JsonObjectBuilder builder, MetricID metricID) {
     private static class CounterImpl implements Counter {
         private final LongAdder adder = new LongAdder();
 
+        private CounterSample counterSample = null;
+
         @Override
         public void inc() {
-            adder.increment();
+            inc(1);
         }
 
         @Override
         public void inc(long n) {
             adder.add(n);
+            counterSample = new CounterSample(n);
         }
 
         @Override
@@ -136,4 +159,11 @@ public String toString() {
                 + "metadata=" + metadata()
                 + '}';
     }
+
+    private static class CounterSample extends LabeledSample<Long> {
+
+        CounterSample(long value) {
+            super(value, ExemplarServiceManager.exemplar());
+        }
+    }
 }