Micrometer metrics (#1137) (#1173)

* [WIP] Micrometer metrics instrumentation

This is a work-in-progress attempt at instrumenting the driver with Micrometer as an optional alternative metrics implementation to the existing internal one.

* Bring it to life.

* Update Micrometer metrics

This update includes general improvements, refactorings and tests.

* Make MetricsAdapter a supported, public api.

- Set scope of micrometer to provided
- Rename MetricsProvider to MetricsAdapter (it does not provide metrics, the driver does. It adapts the drivers metrics to something external)
- Make it a public interface
- Remove dependencies to other internal interfaces and classes (such as BoltServerAddress and the pool itself via ServerAddress and intsuppliers)
- Reduce visibility of internal classes as much as possible
- Make sure that adapters are not serialized
- Apply the clock to the internal adapter (an oversight from the previous iteration)

* Saving a file before commiting seems to be relevant.

* Not messing up the file while saven even more so.

* Make interfaces internal again, provide an enum to select the provider, use global registry by default.

This change keeps the existing serialization feature of config correct.
It will work for everyone just using micrometer as is with the global registry.
if there is a need to select the registriy, we can work on that later.

* Move micrometer-core optional tag to dependency declaration

* Update Config javadoc and remove imports of internal metrics classes

* Update withMetricsEnabled(boolean) implementation

* Enable ConfigBuilder.withMetricsAdapter documentation

* Update driver/src/main/java/org/neo4j/driver/ConnectionPoolMetrics.java

Co-authored-by: Gerrit Meier <meistermeier@gmail.com>

* Update ConnectionPoolMetrics documentation

* Update MetricsAdapter documentation

* Update formatting in ConfigTest

* Update wording

* Updated wording

* Formatting

* Update driver/src/main/java/org/neo4j/driver/internal/metrics/DevNullMetricsListener.java

Co-authored-by: Gerrit Meier <meistermeier@gmail.com>

* Update driver/src/main/java/org/neo4j/driver/internal/metrics/MicrometerConnectionPoolMetrics.java

Co-authored-by: Gerrit Meier <meistermeier@gmail.com>

* Fix compilation error

* Update failing test

* Fix GetConnectionPoolMetrics access

* Delete redundant counters based on review feedback

Co-authored-by: Tommy Ludwig <8924140+shakuzen@users.noreply.github.com>
Co-authored-by: Gerrit Meier <meistermeier@gmail.com>
Co-authored-by: Michael Simons <michael.simons@neo4j.com>

Co-authored-by: Tommy Ludwig <8924140+shakuzen@users.noreply.github.com>
Co-authored-by: Gerrit Meier <meistermeier@gmail.com>
Co-authored-by: Michael Simons <michael.simons@neo4j.com>

bundle/pom.xml

@@ -32,6 +32,11 @@
     </dependency>
 
     <!-- Optional and / or provided dependencies, as they are not transitive to the original driver they must be declared again. -->
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>micrometer-core</artifactId>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

driver/pom.xml

@@ -37,6 +37,11 @@
     </dependency>
 
     <!-- Optional and / or provided dependencies -->
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>micrometer-core</artifactId>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

driver/src/main/java/org/neo4j/driver/Config.java

@@ -36,6 +36,7 @@
 import org.neo4j.driver.internal.handlers.pulln.FetchSizeUtil;
 import org.neo4j.driver.internal.retry.RetrySettings;
 import org.neo4j.driver.net.ServerAddressResolver;
+import org.neo4j.driver.util.Experimental;
 import org.neo4j.driver.util.Immutable;
 
 import static java.lang.String.format;
@@ -96,9 +97,9 @@ public class Config implements Serializable
     private final RetrySettings retrySettings;
     private final ServerAddressResolver resolver;
 
-    private final boolean isMetricsEnabled;
     private final int eventLoopThreads;
     private final String userAgent;
+    private final MetricsAdapter metricsAdapter;
 
     private Config( ConfigBuilder builder )
     {
@@ -122,7 +123,7 @@ private Config( ConfigBuilder builder )
         this.fetchSize = builder.fetchSize;
 
         this.eventLoopThreads = builder.eventLoopThreads;
-        this.isMetricsEnabled = builder.isMetricsEnabled;
+        this.metricsAdapter = builder.metricsAdapter;
     }
 
     /**
@@ -260,7 +261,12 @@ public int eventLoopThreads()
      */
     public boolean isMetricsEnabled()
     {
-        return isMetricsEnabled;
+        return this.metricsAdapter != MetricsAdapter.DEV_NULL;
+    }
+
+    public MetricsAdapter metricsAdapter()
+    {
+        return this.metricsAdapter;
     }
 
     /**
@@ -290,7 +296,7 @@ public static class ConfigBuilder
         private int connectionTimeoutMillis = (int) TimeUnit.SECONDS.toMillis( 30 );
         private RetrySettings retrySettings = RetrySettings.DEFAULT;
         private ServerAddressResolver resolver;
-        private boolean isMetricsEnabled = false;
+        private MetricsAdapter metricsAdapter = MetricsAdapter.DEV_NULL;
         private long fetchSize = FetchSizeUtil.DEFAULT_FETCH_SIZE;
         private int eventLoopThreads = 0;
 
@@ -703,13 +709,13 @@ public ConfigBuilder withResolver( ServerAddressResolver resolver )
         }
 
         /**
-         * Enable driver metrics. The metrics can be obtained afterwards via {@link Driver#metrics()}.
+         * Enable driver metrics backed by internal basic implementation. The metrics can be obtained afterwards via {@link Driver#metrics()}.
+         *
          * @return this builder.
          */
         public ConfigBuilder withDriverMetrics()
         {
-            this.isMetricsEnabled = true;
-            return this;
+            return withMetricsEnabled( true );
         }
 
         /**
@@ -718,7 +724,36 @@ public ConfigBuilder withDriverMetrics()
          */
         public ConfigBuilder withoutDriverMetrics()
         {
-            this.isMetricsEnabled = false;
+            return withMetricsEnabled( false );
+        }
+
+        private ConfigBuilder withMetricsEnabled( boolean enabled )
+        {
+            if ( !enabled )
+            {
+                withMetricsAdapter( MetricsAdapter.DEV_NULL );
+            }
+            else if ( this.metricsAdapter == null || this.metricsAdapter == MetricsAdapter.DEV_NULL )
+            {
+                withMetricsAdapter( MetricsAdapter.DEFAULT );
+            }
+            return this;
+        }
+
+        /**
+         * Enable driver metrics with given {@link MetricsAdapter}.
+         * <p>
+         * {@link MetricsAdapter#MICROMETER} enables implementation based on <a href="https://micrometer.io">Micrometer</a>. The metrics can be obtained
+         * afterwards via Micrometer means and {@link Driver#metrics()}. Micrometer must be on classpath when using this option.
+         * <p>
+         *
+         * @param metricsAdapter the metrics adapter to use. Use {@link MetricsAdapter#DEV_NULL} to disable metrics.
+         * @return this builder.
+         */
+        @Experimental
+        public ConfigBuilder withMetricsAdapter( MetricsAdapter metricsAdapter )
+        {
+            this.metricsAdapter = Objects.requireNonNull( metricsAdapter, "metricsAdapter" );
             return this;
         }
 

driver/src/main/java/org/neo4j/driver/ConnectionPoolMetrics.java

@@ -30,8 +30,9 @@
 public interface ConnectionPoolMetrics
 {
     /**
-     * An unique id that identifies this pool metrics.
-     * @return An unique name
+     * A unique id that identifies this pool metrics.
+     *
+     * @return A unique name
      */
     String id();
 
@@ -57,9 +58,9 @@ public interface ConnectionPoolMetrics
     int creating();
 
     /**
-     * A counter to record how many connections have been successfully created with this pool since the pool is created.
+     * A counter to record how many connections have been successfully created with this pool since the pool was created.
      * This number increases every time when a connection is successfully created.
-     * @return The amount of connections have ever been created by this pool.
+     * @return The amount of connections that have ever been created by this pool.
      */
     long created();
 

driver/src/main/java/org/neo4j/driver/Driver.java

@@ -148,19 +148,17 @@ public interface Driver extends AutoCloseable
 
     /**
      * Returns the driver metrics if metrics reporting is enabled via {@link Config.ConfigBuilder#withDriverMetrics()}.
-     * Otherwise a {@link ClientException} will be thrown.
+     * Otherwise, a {@link ClientException} will be thrown.
      * @return the driver metrics if enabled.
      * @throws ClientException if the driver metrics reporting is not enabled.
      */
-    @Experimental
     Metrics metrics();
 
     /**
      * Returns true if the driver metrics reporting is enabled via {@link Config.ConfigBuilder#withDriverMetrics()}, otherwise false.
      *
      * @return true if the metrics reporting is enabled.
      */
-    @Experimental
     boolean isMetricsEnabled();
 
     /**

driver/src/main/java/org/neo4j/driver/MetricsAdapter.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.driver;
+
+/**
+ * Defines which metrics consumer to use: Should metrics be consumed and exposed via driver's default consumer or provided with one of the external facades.
+ */
+public enum MetricsAdapter
+{
+    /**
+     * Disables metrics.
+     */
+    DEV_NULL,
+
+    /**
+     * Consumes and publishes metrics via the driver itself.
+     */
+    DEFAULT,
+
+    /**
+     * Consumes and publishes metrics via Micrometer. Ensure that Micrometer is on classpath when using this option.
+     */
+    MICROMETER
+}

driver/src/main/java/org/neo4j/driver/internal/DriverFactory.java

@@ -31,6 +31,7 @@
 import org.neo4j.driver.Driver;
 import org.neo4j.driver.Logger;
 import org.neo4j.driver.Logging;
+import org.neo4j.driver.MetricsAdapter;
 import org.neo4j.driver.internal.async.connection.BootstrapFactory;
 import org.neo4j.driver.internal.async.connection.ChannelConnector;
 import org.neo4j.driver.internal.async.connection.ChannelConnectorImpl;
@@ -42,8 +43,10 @@
 import org.neo4j.driver.internal.cluster.loadbalancing.LoadBalancer;
 import org.neo4j.driver.internal.cluster.loadbalancing.LoadBalancingStrategy;
 import org.neo4j.driver.internal.logging.NettyLogging;
+import org.neo4j.driver.internal.metrics.DevNullMetricsProvider;
 import org.neo4j.driver.internal.metrics.InternalMetricsProvider;
 import org.neo4j.driver.internal.metrics.MetricsProvider;
+import org.neo4j.driver.internal.metrics.MicrometerMetricsProvider;
 import org.neo4j.driver.internal.retry.ExponentialBackoffRetryLogic;
 import org.neo4j.driver.internal.retry.RetryLogic;
 import org.neo4j.driver.internal.retry.RetrySettings;
@@ -56,7 +59,6 @@
 
 import static org.neo4j.driver.internal.Scheme.isRoutingScheme;
 import static org.neo4j.driver.internal.cluster.IdentityResolver.IDENTITY_RESOLVER;
-import static org.neo4j.driver.internal.metrics.MetricsProvider.METRICS_DISABLED_PROVIDER;
 import static org.neo4j.driver.internal.util.ErrorUtil.addSuppressed;
 
 public class DriverFactory
@@ -94,7 +96,7 @@ public final Driver newInstance( URI uri, AuthToken authToken, RoutingSettings r
         EventExecutorGroup eventExecutorGroup = bootstrap.config().group();
         RetryLogic retryLogic = createRetryLogic( retrySettings, eventExecutorGroup, config.logging() );
 
-        MetricsProvider metricsProvider = createDriverMetrics( config, createClock() );
+        MetricsProvider metricsProvider = getOrCreateMetricsProvider( config, createClock() );
         ConnectionPool connectionPool = createConnectionPool( authToken, securityPlan, bootstrap, metricsProvider, config,
                                                               ownsEventLoopGroup, newRoutingSettings.routingContext() );
 
@@ -114,16 +116,24 @@ protected ConnectionPool createConnectionPool( AuthToken authToken, SecurityPlan
         return new ConnectionPoolImpl( connector, bootstrap, poolSettings, metricsProvider.metricsListener(), config.logging(), clock, ownsEventLoopGroup );
     }
 
-    protected static MetricsProvider createDriverMetrics( Config config, Clock clock )
+    protected static MetricsProvider getOrCreateMetricsProvider( Config config, Clock clock )
     {
-        if( config.isMetricsEnabled() )
+        MetricsAdapter metricsAdapter = config.metricsAdapter();
+        // This can actually only happen when someone mocks the config
+        if ( metricsAdapter == null )
         {
-            return new InternalMetricsProvider( clock, config.logging() );
+            metricsAdapter = config.isMetricsEnabled() ? MetricsAdapter.DEFAULT : MetricsAdapter.DEV_NULL;
         }
-        else
+        switch ( metricsAdapter )
         {
-            return METRICS_DISABLED_PROVIDER;
+        case DEV_NULL:
+            return DevNullMetricsProvider.INSTANCE;
+        case DEFAULT:
+            return new InternalMetricsProvider( clock, config.logging() );
+        case MICROMETER:
+            return MicrometerMetricsProvider.forGlobalRegistry();
         }
+        throw new IllegalStateException( "Unknown or unsupported MetricsAdapter: " + metricsAdapter );
     }
 
     protected ChannelConnector createConnector( ConnectionSettings settings, SecurityPlan securityPlan,

driver/src/main/java/org/neo4j/driver/internal/InternalDriver.java

@@ -25,12 +25,13 @@
 import org.neo4j.driver.Logger;
 import org.neo4j.driver.Logging;
 import org.neo4j.driver.Metrics;
+import org.neo4j.driver.internal.metrics.MetricsProvider;
 import org.neo4j.driver.Session;
 import org.neo4j.driver.SessionConfig;
 import org.neo4j.driver.async.AsyncSession;
 import org.neo4j.driver.internal.async.InternalAsyncSession;
 import org.neo4j.driver.internal.async.NetworkSession;
-import org.neo4j.driver.internal.metrics.MetricsProvider;
+import org.neo4j.driver.internal.metrics.DevNullMetricsProvider;
 import org.neo4j.driver.internal.reactive.InternalRxSession;
 import org.neo4j.driver.internal.security.SecurityPlan;
 import org.neo4j.driver.internal.types.InternalTypeSystem;
@@ -102,7 +103,7 @@ public Metrics metrics()
     @Override
     public boolean isMetricsEnabled()
     {
-        return metricsProvider.isMetricsEnabled();
+        return metricsProvider != DevNullMetricsProvider.INSTANCE;
     }
 
     @Override

driver/src/main/java/org/neo4j/driver/internal/async/pool/ConnectionPoolImpl.java

@@ -47,6 +47,7 @@
 import org.neo4j.driver.internal.spi.ConnectionPool;
 import org.neo4j.driver.internal.util.Clock;
 import org.neo4j.driver.internal.util.Futures;
+import org.neo4j.driver.net.ServerAddress;
 
 import static java.lang.String.format;
 import static org.neo4j.driver.internal.async.connection.ChannelAttributes.setAuthorizationStateListener;
@@ -159,13 +160,13 @@ public void retainAll( Set<BoltServerAddress> addressesToRetain )
     }
 
     @Override
-    public int inUseConnections( BoltServerAddress address )
+    public int inUseConnections( ServerAddress address )
     {
         return nettyChannelTracker.inUseChannelCount( address );
     }
 
     @Override
-    public int idleConnections( BoltServerAddress address )
+    public int idleConnections( ServerAddress address )
     {
         return nettyChannelTracker.idleChannelCount( address );
     }
@@ -284,8 +285,8 @@ private ExtendedChannelPool getOrCreatePool( BoltServerAddress address )
                                       if ( pool == null )
                                       {
                                           pool = newPool( address );
-                                          // before the connection pool is added I can add the metrics for the pool.
-                                          metricsListener.putPoolMetrics( pool.id(), address, this );
+                                          // before the connection pool is added I can register the metrics for the pool.
+                                          metricsListener.registerPoolMetrics( pool.id(), address, () -> this.inUseConnections( address ), () -> this.idleConnections( address ) );
                                           addressToPool.put( address, pool );
                                       }
                                       return pool;

driver/src/main/java/org/neo4j/driver/internal/async/pool/NettyChannelTracker.java

@@ -37,6 +37,7 @@
 import org.neo4j.driver.internal.messaging.BoltProtocol;
 import org.neo4j.driver.internal.metrics.ListenerEvent;
 import org.neo4j.driver.internal.metrics.MetricsListener;
+import org.neo4j.driver.net.ServerAddress;
 
 import static org.neo4j.driver.internal.async.connection.ChannelAttributes.poolId;
 import static org.neo4j.driver.internal.async.connection.ChannelAttributes.serverAddress;
@@ -46,8 +47,8 @@ public class NettyChannelTracker implements ChannelPoolHandler
     private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     private final Lock read = lock.readLock();
     private final Lock write = lock.writeLock();
-    private final Map<BoltServerAddress,Integer> addressToInUseChannelCount = new HashMap<>();
-    private final Map<BoltServerAddress,Integer> addressToIdleChannelCount = new HashMap<>();
+    private final Map<ServerAddress,Integer> addressToInUseChannelCount = new HashMap<>();
+    private final Map<ServerAddress,Integer> addressToIdleChannelCount = new HashMap<>();
     private final Logger log;
     private final MetricsListener metricsListener;
     private final ChannelFutureListener closeListener = future -> channelClosed( future.channel() );
@@ -152,12 +153,12 @@ public void channelClosed( Channel channel )
         metricsListener.afterClosed( poolId( channel ) );
     }
 
-    public int inUseChannelCount( BoltServerAddress address )
+    public int inUseChannelCount( ServerAddress address )
     {
         return retrieveInReadLock( () -> addressToInUseChannelCount.getOrDefault( address, 0 ) );
     }
 
-    public int idleChannelCount( BoltServerAddress address )
+    public int idleChannelCount( ServerAddress address )
     {
         return retrieveInReadLock( () -> addressToIdleChannelCount.getOrDefault( address, 0 ) );
     }
@@ -213,9 +214,9 @@ private void decrementIdle( Channel channel )
         addressToIdleChannelCount.put( address, count - 1 );
     }
 
-    private void increment( Channel channel, Map<BoltServerAddress,Integer> countMap )
+    private void increment( Channel channel, Map<ServerAddress,Integer> countMap )
     {
-        BoltServerAddress address = serverAddress( channel );
+        ServerAddress address = serverAddress( channel );
         Integer count = countMap.computeIfAbsent( address, k -> 0 );
         countMap.put( address, count + 1 );
     }