feat: add prometheus metrics for caches reverting telemetry metrics

baseapp/options.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/line/lbm-sdk/v2/store/cache"
 	"github.com/line/lbm-sdk/v2/store/iavl"
 	tmdb "github.com/line/tm-db/v2"
 
@@ -65,12 +66,12 @@ func SetInterBlockCache(cache sdk.MultiStorePersistentCache) func(*BaseApp) {
 }
 
 // SetIAVLCacheManager provides a BaseApp option function that sets the iavl CacheManager
-func SetIAVLCacheManager(size int) func(*BaseApp) {
+func SetIAVLCacheManager(size int, provider iavl.MetricsProvider) func(*BaseApp) {
 	return func(app *BaseApp) {
 		if size == 0 {
 			app.cms.SetIAVLCacheManager(iavl.NewCacheManagerNoCache())
 		} else {
-			app.cms.SetIAVLCacheManager(iavl.NewCacheManagerSingleton(size))
+			app.cms.SetIAVLCacheManager(iavl.NewCacheManagerSingleton(size, provider))
 		}
 	}
 }
@@ -245,3 +246,11 @@ func (app *BaseApp) SetInterfaceRegistry(registry types.InterfaceRegistry) {
 	app.grpcQueryRouter.SetInterfaceRegistry(registry)
 	app.msgServiceRouter.SetInterfaceRegistry(registry)
 }
+
+func MetricsProvider(prometheus bool) (cache.MetricsProvider, iavl.MetricsProvider) {
+	namespace := "app"
+	if prometheus {
+		return cache.PrometheusMetricsProvider(namespace), iavl.PrometheusMetricsProvider(namespace)
+	}
+	return cache.NopMetricsProvider(), iavl.NopMetricsProvider()
+}

baseapp/options_test.go

@@ -0,0 +1,22 @@
+package baseapp
+
+import (
+	"testing"
+
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetricsProvider(t *testing.T) {
+	p1, p2 := MetricsProvider(true)
+	c1 := p1()
+	c2 := p2()
+	require.NotEqual(t, c1.InterBlockCacheHits, discard.NewCounter())
+	require.NotEqual(t, c2.IAVLCacheHits, discard.NewGauge())
+
+	p1, p2 = MetricsProvider(false)
+	c1 = p1()
+	c2 = p2()
+	require.Equal(t, c1.InterBlockCacheHits, discard.NewCounter())
+	require.Equal(t, c2.IAVLCacheHits, discard.NewGauge())
+}

server/config/config.go

@@ -73,6 +73,10 @@ type BaseConfig struct {
 
 	// IAVL cache size; bytes size unit
 	IAVLCacheSize int `mapstructure:"iavl-cache-size"`
+
+	// When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
+	// It works when tendermint's prometheus option (config.toml) is set to true.
+	Prometheus bool `mapstructure:"prometheus"`
 }
 
 // APIConfig defines the API listener configuration.

server/config/toml.go

@@ -76,6 +76,10 @@ iavl-cache-size = {{ .BaseConfig.IAVLCacheSize }}
 # ["message.sender", "message.recipient"]
 index-events = {{ .BaseConfig.IndexEvents }}
 
+# When true, Prometheus metrics are served under /metrics on prometheus_listen_addr in config.toml.
+# It works when tendermint's prometheus option (config.toml) is set to true.
+prometheus = {{ .BaseConfig.Prometheus }}
+
 ###############################################################################
 ###                         Telemetry Configuration                         ###
 ###############################################################################

server/start.go

@@ -160,6 +160,8 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval")
 	cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep")
 
+	cmd.Flags().Bool(FlagPrometheus, false, "Enable prometheus metric for app")
+
 	// add support for all Ostracon-specific command line options
 	ostcmd.AddNodeFlags(cmd)
 	return cmd

simapp/sim_test.go

@@ -54,7 +54,8 @@ func fauxMerkleModeOpt(bapp *baseapp.BaseApp) {
 // interBlockCacheOpt returns a BaseApp option function that sets the persistent
 // inter-block write-through cache.
 func interBlockCacheOpt() func(*baseapp.BaseApp) {
-	return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize))
+	return baseapp.SetInterBlockCache(store.NewCommitKVStoreCacheManager(
+		cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider()))
 }
 
 func TestFullAppSimulation(t *testing.T) {

simapp/simd/cmd/root.go

@@ -6,6 +6,13 @@ import (
 	"os"
 	"path/filepath"
 
+	ostcli "github.com/line/ostracon/libs/cli"
+	"github.com/line/ostracon/libs/log"
+	tmdb "github.com/line/tm-db/v2"
+	"github.com/spf13/cast"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
 	"github.com/line/lbm-sdk/v2/baseapp"
 	"github.com/line/lbm-sdk/v2/client"
 	"github.com/line/lbm-sdk/v2/client/debug"
@@ -26,11 +33,6 @@ import (
 	banktypes "github.com/line/lbm-sdk/v2/x/bank/types"
 	"github.com/line/lbm-sdk/v2/x/crisis"
 	genutilcli "github.com/line/lbm-sdk/v2/x/genutil/client/cli"
-	ostcli "github.com/line/ostracon/libs/cli"
-	"github.com/line/ostracon/libs/log"
-	tmdb "github.com/line/tm-db/v2"
-	"github.com/spf13/cast"
-	"github.com/spf13/cobra"
 )
 
 // NewRootCmd creates a new root command for simd. It is called once in the
@@ -156,9 +158,11 @@ type appCreator struct {
 func (a appCreator) newApp(logger log.Logger, db tmdb.DB, traceStore io.Writer, appOpts servertypes.AppOptions) servertypes.Application {
 	var cache sdk.MultiStorePersistentCache
 
+	ibCacheMetricsProvider, iavlCacheMetricsProvider :=
+		baseapp.MetricsProvider(cast.ToBool(viper.GetBool(server.FlagPrometheus)))
 	if cast.ToBool(appOpts.Get(server.FlagInterBlockCache)) {
 		cache = store.NewCommitKVStoreCacheManager(
-			cast.ToInt(appOpts.Get(server.FlagInterBlockCacheSize)))
+			cast.ToInt(appOpts.Get(server.FlagInterBlockCacheSize)), ibCacheMetricsProvider)
 	}
 
 	skipUpgradeHeights := make(map[int64]bool)
@@ -192,10 +196,10 @@ func (a appCreator) newApp(logger log.Logger, db tmdb.DB, traceStore io.Writer,
 		baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),
 		baseapp.SetHaltTime(cast.ToUint64(appOpts.Get(server.FlagHaltTime))),
 		baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))),
-		baseapp.SetIAVLCacheManager(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize))),
 		baseapp.SetInterBlockCache(cache),
-		baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
+		baseapp.SetIAVLCacheManager(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize)), iavlCacheMetricsProvider),
 		baseapp.SetTrace(cast.ToBool(appOpts.Get(server.FlagTrace))),
+		baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),
 		baseapp.SetSnapshotStore(snapshotStore),
 		baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))),
 		baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))),

store/cache/cache.go

@@ -6,7 +6,6 @@ import (
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/line/lbm-sdk/v2/store/cachekv"
 	"github.com/line/lbm-sdk/v2/store/types"
-	"github.com/line/lbm-sdk/v2/telemetry"
 )
 
 const (
@@ -27,18 +26,20 @@ type (
 	// CommitKVStore and below is completely irrelevant to this layer.
 	CommitKVStoreCache struct {
 		types.CommitKVStore
-		cache  *fastcache.Cache
-		prefix []byte
+		cache   *fastcache.Cache
+		prefix  []byte
+		metrics *Metrics
 	}
 
 	// CommitKVStoreCacheManager maintains a mapping from a StoreKey to a
 	// CommitKVStoreCache. Each CommitKVStore, per StoreKey, is meant to be used
 	// in an inter-block (persistent) manner and typically provided by a
 	// CommitMultiStore.
 	CommitKVStoreCacheManager struct {
-		mutex  sync.Mutex
-		cache  *fastcache.Cache
-		caches map[string]types.CommitKVStore
+		mutex   sync.Mutex
+		cache   *fastcache.Cache
+		caches  map[string]types.CommitKVStore
+		metrics *Metrics
 
 		// All cache stores use the unique prefix that has one byte length
 		// Contract: The number of all cache stores cannot exceed 127(max byte)
@@ -47,22 +48,25 @@ type (
 	}
 )
 
-func NewCommitKVStoreCache(store types.CommitKVStore, prefix []byte, cache *fastcache.Cache) *CommitKVStoreCache {
+func NewCommitKVStoreCache(store types.CommitKVStore, prefix []byte, cache *fastcache.Cache,
+	metrics *Metrics) *CommitKVStoreCache {
 	return &CommitKVStoreCache{
 		CommitKVStore: store,
 		prefix:        prefix,
 		cache:         cache,
+		metrics:       metrics,
 	}
 }
 
-func NewCommitKVStoreCacheManager(cacheSize int) *CommitKVStoreCacheManager {
+func NewCommitKVStoreCacheManager(cacheSize int, provider MetricsProvider) *CommitKVStoreCacheManager {
 	if cacheSize <= 0 {
 		// This function was called because it intended to use the inter block cache, creating a cache of minimal size.
 		cacheSize = DefaultCommitKVStoreCacheSize
 	}
 	return &CommitKVStoreCacheManager{
 		cache:       fastcache.New(cacheSize),
 		caches:      make(map[string]types.CommitKVStore),
+		metrics:     provider(),
 		prefixMap:   make(map[string][]byte),
 		prefixOrder: 0,
 	}
@@ -81,7 +85,7 @@ func (cmgr *CommitKVStoreCacheManager) GetStoreCache(key types.StoreKey, store t
 			if cmgr.prefixOrder <= 0 {
 				panic("The number of cache stores exceed the maximum(127)")
 			}
-			cmgr.caches[key.Name()] = NewCommitKVStoreCache(store, cmgr.prefixMap[key.Name()], cmgr.cache)
+			cmgr.caches[key.Name()] = NewCommitKVStoreCache(store, cmgr.prefixMap[key.Name()], cmgr.cache, cmgr.metrics)
 		}
 		cmgr.mutex.Unlock()
 	}
@@ -123,18 +127,18 @@ func (ckv *CommitKVStoreCache) Get(key []byte) []byte {
 	valueI := ckv.cache.Get(nil, prefixedKey)
 	if valueI != nil {
 		// cache hit
-		telemetry.IncrCounter(1, "store", "inter-block-cache", "hits")
+		ckv.metrics.InterBlockCacheHits.Add(1)
 		return valueI
 	}
 
 	// cache miss; write to cache
-	telemetry.IncrCounter(1, "store", "inter-block-cache", "misses")
+	ckv.metrics.InterBlockCacheMisses.Add(1)
 	value := ckv.CommitKVStore.Get(key)
 	ckv.cache.Set(prefixedKey, value)
 	stats := fastcache.Stats{}
 	ckv.cache.UpdateStats(&stats)
-	telemetry.SetGauge(float32(stats.EntriesCount), "store", "inter-block-cache", "entries")
-	telemetry.SetGauge(float32(stats.BytesSize), "store", "inter-block-cache", "bytes")
+	ckv.metrics.InterBlockCacheEntries.Set(float64(stats.EntriesCount))
+	ckv.metrics.InterBlockCacheBytes.Set(float64(stats.BytesSize))
 	return value
 }
 

store/cache/cache_test.go

@@ -16,7 +16,7 @@ import (
 
 func TestGetOrSetStoreCache(t *testing.T) {
 	db := memdb.NewDB()
-	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
+	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
 
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)
@@ -30,7 +30,7 @@ func TestGetOrSetStoreCache(t *testing.T) {
 
 func TestUnwrap(t *testing.T) {
 	db := memdb.NewDB()
-	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
+	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
 
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)
@@ -44,7 +44,7 @@ func TestUnwrap(t *testing.T) {
 
 func TestStoreCache(t *testing.T) {
 	db := memdb.NewDB()
-	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
+	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize, cache.NopMetricsProvider())
 
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)

store/cache/metrics.go

@@ -0,0 +1,84 @@
+package cache
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "inter_block_cache"
+)
+
+// Metrics contains metrics exposed by this package.
+type Metrics struct {
+	InterBlockCacheHits    metrics.Counter
+	InterBlockCacheMisses  metrics.Counter
+	InterBlockCacheEntries metrics.Gauge
+	InterBlockCacheBytes   metrics.Gauge
+}
+
+// PrometheusMetrics returns Metrics build using Prometheus client library.
+// Optionally, labels can be provided along with their values ("foo",
+// "fooValue").
+func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(labelsAndValues); i += 2 {
+		labels = append(labels, labelsAndValues[i])
+	}
+	return &Metrics{
+		InterBlockCacheHits: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "hits",
+			Help:      "Cache hits of the inter block cache",
+		}, labels).With(labelsAndValues...),
+		InterBlockCacheMisses: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "misses",
+			Help:      "Cache misses of the inter block cache",
+		}, labels).With(labelsAndValues...),
+		InterBlockCacheEntries: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "entries",
+			Help:      "Cache entry count of the inter block cache",
+		}, labels).With(labelsAndValues...),
+		InterBlockCacheBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "bytes_size",
+			Help:      "Cache bytes size of the inter block cache",
+		}, labels).With(labelsAndValues...),
+	}
+}
+
+// NopMetrics returns no-op Metrics.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		InterBlockCacheHits:    discard.NewCounter(),
+		InterBlockCacheMisses:  discard.NewCounter(),
+		InterBlockCacheEntries: discard.NewGauge(),
+		InterBlockCacheBytes:   discard.NewGauge(),
+	}
+}
+
+type MetricsProvider func() *Metrics
+
+// PrometheusMetricsProvider returns PrometheusMetrics for each store
+func PrometheusMetricsProvider(namespace string, labelsAndValues ...string) func() *Metrics {
+	return func() *Metrics {
+		return PrometheusMetrics(namespace, labelsAndValues...)
+	}
+}
+
+// NopMetricsProvider returns NopMetrics for each store
+func NopMetricsProvider() func() *Metrics {
+	return func() *Metrics {
+		return NopMetrics()
+	}
+}

store/cache/metrics_test.go

@@ -0,0 +1,16 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrometheusMetrics(t *testing.T) {
+	metrics := PrometheusMetrics("test")
+	require.NotEqual(t, metrics.InterBlockCacheHits, discard.NewCounter())
+	require.NotEqual(t, metrics.InterBlockCacheMisses, discard.NewCounter())
+	require.NotEqual(t, metrics.InterBlockCacheEntries, discard.NewGauge())
+	require.NotEqual(t, metrics.InterBlockCacheBytes, discard.NewGauge())
+}