Add exemplar support to histogram (and histogram part of timer) Prometheus output

bom/pom.xml

@@ -512,6 +512,11 @@
                 <artifactId>helidon-metrics</artifactId>
                 <version>${helidon.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.helidon.metrics</groupId>
+                <artifactId>helidon-metrics-trace-exemplar</artifactId>
+                <version>${helidon.version}</version>
+            </dependency>
 
             <!-- health checks -->
             <dependency>

docs/mp/metrics/04_prometheus_exemplar_support.adoc

@@ -0,0 +1,25 @@
+///////////////////////////////////////////////////////////////////////////////
+
+    Copyright (c) 2021 Oracle and/or its affiliates.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+///////////////////////////////////////////////////////////////////////////////
+
+= Metrics Support for Exemplars
+
+:h1Prefix: MP
+:description: Helidon metrics
+:keywords: helidon, metrics, exemplar, prometheus, OpenMetrics
+
+include::../../shared/metrics/prometheus-exemplar-support.adoc[]

docs/se/metrics/04_prometheus_exemplar_support.adoc

@@ -0,0 +1,25 @@
+///////////////////////////////////////////////////////////////////////////////
+
+    Copyright (c) 2021 Oracle and/or its affiliates.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+///////////////////////////////////////////////////////////////////////////////
+
+= Metrics Support for Exemplars
+
+:h1Prefix: SE
+:description: Helidon metrics
+:keywords: helidon, metrics, exemplar, prometheus, OpenMetrics
+
+include::../../shared/metrics/prometheus-exemplar-support.adoc[]

docs/shared/metrics/prometheus-exemplar-support.adoc

@@ -0,0 +1,99 @@
+///////////////////////////////////////////////////////////////////////////////
+
+    Copyright (c) 2021 Oracle and/or its affiliates.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+///////////////////////////////////////////////////////////////////////////////
+
+:description: Helidon metrics
+:keywords: helidon, metrics, exemplar, prometheus, OpenMetrics
+:zipkin-page: ../../shared/tracing/tracer-zipkin.adoc
+:jaeger-page: ../../shared/tracing/tracer-jaeger.adoc
+:open-metrics-spec-exemplars: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
+
+Add Helidon {h1Prefix} support for  link:{open-metrics-spec-exemplars}[OpenMetrics (Prometheus) exemplars] for histograms, counters, and simple timers to your application simply by adding dependencies to your project's `pom.xml`.
+
+== Prerequisites
+
+Declare the following dependency in your project:
+
+[source,xml,subs="verbatim,attributes"]
+----
+<dependency>
+    <groupId>io.helidon.metrics</groupId>
+    <artifactId>helidon-metrics-trace-exemplar</artifactId>
+    <scope>runtime</scope>
+</dependency>
+----
+
+Also, include either link:{zipkin-page}[Helidon Zipkin] or link:{jaeger-page}[Helidon Jaeger] support:
+include::{zipkin-page}[tag=zipkin-dependency]
+or
+include::{jaeger-page}[tag=jaeger-dependency]
+
+Be sure Zipkin or Jaeger, whichever you chose, is running and accessible to your server.
+
+== Interpreting Exemplars
+
+//[quote,Mirriam-Webster Dictionary,'link:https://www.merriam-webster.com/dictionary/exemplar[exemplar]'']
+//____
+//exemplar - one that serves as a model or example
+//____
+[NOTE]
+--
+link:https://www.merriam-webster.com/dictionary/exemplar[_exemplar_] - one that serves as a model or example
+[.text-right]
+-- Merriam-Webster Dictionary
+--
+
+When you add the `helidon-metrics-trace-exemplar` dependency--and one for either Zipkin or Jaeger--to your application, Helidon automatically records a sample (label, value, and timestamp) with each update to a histogram, simple timer, or counter. Helidon adds the label, value, and timestamp to the OpenMetrics output returned from the Helidon metrics endpoint (`/metrics` unless you set it up otherwise).
+
+.Exemplar output - `Timer`
+[listing]
+----
+# TYPE application_getTimer_mean_seconds gauge
+application_getTimer_mean_seconds 8.303030623354298E-4 # {trace_id="067632454fe4e8d1"} 1.14701E-4 1617723032.570000 <1>
+
+# TYPE application_getTimer_max_seconds gauge
+application_getTimer_max_seconds 0.003952636 # {trace_id="fce183094e471633"} 0.003952636 1617723030.108000 <2>
+
+# TYPE application_getTimer_min_seconds gauge
+application_getTimer_min_seconds 5.5254E-5 # {trace_id="0b1a4bf22b4e47fd"} 5.5254E-5 1617723033.311000
+----
+<1> This exemplar is a sample with value at least as close to the mean as any other sample.
+<2> This exemplar is for an exact sample with value the same as the maximum value the timer has observed.
+
+.Exemplar output - `SimpleTimer`
+[listing]
+----
+# TYPE application_globalRequestTracker_total counter
+# HELP application_globalRequestTracker_total
+application_globalRequestTracker_total 4 # {trace_id="daf26fe35fee9917"} 0.001183992 1617725180.234000 <1>
+
+# TYPE application_globalRequestTracker_elapsedTime_seconds gauge
+application_globalRequestTracker_elapsedTime_seconds 0.030309068 # {trace_id="daf26fe35fee9917"} 0.001183992 1617725180.234000 <1>
+----
+<1> The exemplar for a `SimpleTimer` is the same for the `total` and the `elapsedTime` submetrics: always the most recent sample which updated the `SimpleTimer`.
+
+Helidon adds an exemplar to the output for each statistical value--such as minimum, maximum, mean, and quantiles--for histograms, timers, simple times, and for counters. The exemplar information describes a single, actual sample that is representative of the statistical value.
+Helidon chooses the representative examplar for each value using information that is already recorded for each type of metric:
+
+. If a metric necessarily corresponds to a specific sample--for example a minimum or maximum--Helidon associates a sample that has that exact value as the exemmplar for the metric.
+. If a metric collects samples into bins (quantiles), Helidon associates a sample from that bin with the bin's output.
+. If a metric maintains running statistics (counts, totals), Helidon associates the most recent sample for that metric.
+. If Helidon computes a metric's value from a number of samples--for example, mean--Helidon associates a sample for which its value is at least as close as other samples to the statistical calculation.
+
+In cases with multiple representative samples (for example, two samples' values are equally close to the mean), Helidon chooses one of them arbitrarily.
+
+

docs/shared/tracing/tracer-jaeger.adoc

@@ -1,6 +1,6 @@
 ///////////////////////////////////////////////////////////////////////////////
 
-    Copyright (c) 2019, 2020 Oracle and/or its affiliates.
+    Copyright (c) 2019, 2021 Oracle and/or its affiliates.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -27,14 +27,15 @@ on the Jaeger tracer.
 To use Jaeger as a tracer,
     add the following dependency to your project:
 
+// tag::jaeger-dependency[]
 [source,xml]
 ----
 <dependency>
     <groupId>io.helidon.tracing</groupId>
     <artifactId>helidon-tracing-jaeger</artifactId>
 </dependency>
 ----
-
+// end::jaeger-dependency[]
 == Configuring Jaeger
 
 The Jaeger tracer supports the following configuration options:

docs/shared/tracing/tracer-zipkin.adoc

@@ -1,6 +1,6 @@
 ///////////////////////////////////////////////////////////////////////////////
 
-    Copyright (c) 2019, 2020 Oracle and/or its affiliates.
+    Copyright (c) 2019, 2021 Oracle and/or its affiliates.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -27,13 +27,15 @@ on the Zipkin tracer.
 To use Zipkin as a tracer,
     add the following dependency to your project:
 
+// tag::zipkin-dependency[]
 [source,xml]
 ----
 <dependency>
     <groupId>io.helidon.tracing</groupId>
     <artifactId>helidon-tracing-zipkin</artifactId>
 </dependency>
 ----
+// end::zipkin-dependency[]
 
 == Configuring Zipkin
 

examples/metrics/exemplar/README.md

@@ -0,0 +1,68 @@
+# Helidon Metrics Exemplar SE Example
+
+This project implements a simple Hello World REST service using Helidon SE and demonstrates the 
+optional metrics exemplar support.
+
+## Start Zipkin (optional)
+If you do not start Zipkin, the example app will still function correctly but it will log a warning 
+when it cannot contact the Zipkin server to report the tracing spans. Even so, the metrics output 
+will contain valid exemplars.
+
+With Docker:
+```bash
+docker run --name zipkin -d -p 9411:9411 openzipkin/zipkin
+```
+
+## Build and run
+
+With JDK11+
+```bash
+mvn package
+java -jar target/helidon-examples-metrics-exemplar.jar
+```
+
+## Exercise the application
+
+```
+curl -X GET http://localhost:8080/greet
+{"message":"Hello World!"}
+
+curl -X GET http://localhost:8080/greet/Joe
+{"message":"Hello Joe!"}
+
+curl -X PUT -H "Content-Type: application/json" -d '{"greeting" : "Hola"}' http://localhost:8080/greet/greeting
+
+curl -X GET http://localhost:8080/greet/Jose
+{"message":"Hola Jose!"}
+
+curl -X GET http://localhost:8080/greet          
+{"message":"Hola World!"}
+```
+
+## Retrieve application metrics
+
+```
+# Prometheus format with exemplars
+
+curl -s -X GET http://localhost:8080/metrics/application
+# TYPE application_counterForPersonalizedGreetings_total counter
+# HELP application_counterForPersonalizedGreetings_total 
+application_counterForPersonalizedGreetings_total 2 # {trace_id="78e61eed351f4c9d"} 1 1617812495.016000
+. . .
+# TYPE application_timerForGets_mean_seconds gauge
+application_timerForGets_mean_seconds 0.005772598385062112 # {trace_id="b22f13c37ba8b879"} 0.001563945 1617812578.687000
+# TYPE application_timerForGets_max_seconds gauge
+application_timerForGets_max_seconds 0.028018165 # {trace_id="a1b127002725143c"} 0.028018165 1617812467.524000
+```
+The examplars contain `trace_id` values tying them to specific samples.
+Note that the exemplar for the counter refers to the most recent update to the counter. 
+
+For the timer, the value for the `max` is exactly the same as the value for its exemplar, 
+because the `max` value has to come from at least one sample. 
+In contrast, Helidon calculates the `mean` value from possibly multiple samples. The exemplar for 
+`mean` is a sample with value as close as that of other samples to the mean. 
+
+## Browse the Zipkin traces
+If you started the Zipkin server, visit `http://localhost:9411` and click `Run Query` to see all 
+the spans your Helidon application reported to Zipkin.
+You can compare the trace IDs in the Zipkin display to those in the metrics output.

examples/metrics/exemplar/pom.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright (c) 2021 Oracle and/or its affiliates.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+--><project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>io.helidon.applications</groupId>
+        <artifactId>helidon-se</artifactId>
+        <version>2.3.0-SNAPSHOT</version>
+        <relativePath>../../../applications/se/pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>io.helidon.examples.metrics</groupId>
+    <artifactId>helidon-examples-metrics-exemplar</artifactId>
+
+    <name>Helidon Examples Metrics Exemplar</name>
+
+    <properties>
+        <mainClass>io.helidon.examples.metrics.exemplar.Main</mainClass>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.helidon.webserver</groupId>
+            <artifactId>helidon-webserver</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.media</groupId>
+            <artifactId>helidon-media-jsonp</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.metrics</groupId>
+            <artifactId>helidon-metrics</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.tracing</groupId>
+            <artifactId>helidon-tracing</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.metrics</groupId>
+            <artifactId>helidon-metrics-trace-exemplar</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.tracing</groupId>
+            <artifactId>helidon-tracing-zipkin</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.helidon.webclient</groupId>
+            <artifactId>helidon-webclient</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-libs</id>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>