From 7d34b9fb3f427f065f2e8e827564beff9367d2af Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Thu, 2 Dec 2021 14:56:31 -0500 Subject: [PATCH] Add metrics emission surrounding boltdb performance (#26) --- bolt_store.go | 11 ++++++++ go.mod | 3 +- go.sum | 71 +++++++++++++++++++++++++++++++++++++++++++++++- metrics.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++ v2/bolt_store.go | 12 ++++++++ v2/go.mod | 1 + v2/metrics.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 metrics.go create mode 100644 v2/metrics.go diff --git a/bolt_store.go b/bolt_store.go index 66b75bc..19792eb 100644 --- a/bolt_store.go +++ b/bolt_store.go @@ -2,7 +2,9 @@ package raftboltdb import ( "errors" + "time" + metrics "github.com/armon/go-metrics" "github.com/boltdb/bolt" "github.com/hashicorp/raft" ) @@ -166,24 +168,33 @@ func (b *BoltStore) StoreLog(log *raft.Log) error { // StoreLogs is used to store a set of raft logs func (b *BoltStore) StoreLogs(logs []*raft.Log) error { + defer metrics.MeasureSince([]string{"raft", "boltdb", "storeLogs"}, time.Now()) tx, err := b.conn.Begin(true) if err != nil { return err } defer tx.Rollback() + batchSize := 0 for _, log := range logs { key := uint64ToBytes(log.Index) val, err := encodeMsgPack(log) if err != nil { return err } + + logLen := val.Len() bucket := tx.Bucket(dbLogs) if err := bucket.Put(key, val.Bytes()); err != nil { return err } + batchSize += logLen + metrics.AddSample([]string{"raft", "boltdb", "logSize"}, float32(logLen)) } + metrics.AddSample([]string{"raft", "boltdb", "logsPerBatch"}, float32(len(logs))) + metrics.AddSample([]string{"raft", "boltdb", "logBatchSize"}, float32(batchSize)) + return tx.Commit() } diff --git a/go.mod b/go.mod index 65f1577..e34d574 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/hashicorp/raft-boltdb go 1.12 require ( + github.com/armon/go-metrics v0.3.8 // indirect github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 - ) +) diff --git a/go.sum b/go.sum index ca41d0a..1abf693 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,36 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.3.8 h1:oOxq3KPj0WhCuy50EhzwiyMyG2ovRQZpZLXQuOh2a/M= +github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -24,20 +45,68 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= \ No newline at end of file +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..9cb7eb8 --- /dev/null +++ b/metrics.go @@ -0,0 +1,68 @@ +package raftboltdb + +import ( + "context" + "time" + + metrics "github.com/armon/go-metrics" + "github.com/boltdb/bolt" +) + +const ( + defaultMetricsInterval = 5 * time.Second +) + +// RunMetrics should be executed in a go routine and will periodically emit +// metrics on the given interval until the context has been cancelled. +func (b *BoltStore) RunMetrics(ctx context.Context, interval time.Duration) { + if interval == 0 { + interval = defaultMetricsInterval + } + + tick := time.NewTicker(interval) + defer tick.Stop() + + stats := b.emitMetrics(nil) + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + stats = b.emitMetrics(stats) + } + } +} + +func (b *BoltStore) emitMetrics(prev *bolt.Stats) *bolt.Stats { + newStats := b.conn.Stats() + + stats := newStats + if prev != nil { + stats = newStats.Sub(prev) + } + + // freelist metrics + metrics.SetGauge([]string{"raft", "boltdb", "numFreePages"}, float32(newStats.FreePageN)) + metrics.SetGauge([]string{"raft", "boltdb", "numPendingPages"}, float32(newStats.PendingPageN)) + metrics.SetGauge([]string{"raft", "boltdb", "freePageBytes"}, float32(newStats.FreeAlloc)) + metrics.SetGauge([]string{"raft", "boltdb", "freelistBytes"}, float32(newStats.FreelistInuse)) + + // txn metrics + metrics.IncrCounter([]string{"raft", "boltdb", "totalReadTxn"}, float32(stats.TxN)) + metrics.SetGauge([]string{"raft", "boltdb", "openReadTxn"}, float32(newStats.OpenTxN)) + + // tx stats + metrics.SetGauge([]string{"raft", "boltdb", "txstats", "pageCount"}, float32(newStats.TxStats.PageCount)) + metrics.SetGauge([]string{"raft", "boltdb", "txstats", "pageAlloc"}, float32(newStats.TxStats.PageAlloc)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "cursorCount"}, float32(stats.TxStats.CursorCount)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "nodeCount"}, float32(stats.TxStats.NodeCount)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "nodeDeref"}, float32(stats.TxStats.NodeDeref)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "rebalance"}, float32(stats.TxStats.Rebalance)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "rebalanceTime"}, float32(stats.TxStats.RebalanceTime.Nanoseconds())/1000000) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "split"}, float32(stats.TxStats.Split)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "spill"}, float32(stats.TxStats.Spill)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "spillTime"}, float32(stats.TxStats.SpillTime.Nanoseconds())/1000000) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "write"}, float32(stats.TxStats.Write)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "writeTime"}, float32(stats.TxStats.WriteTime.Nanoseconds())/1000000) + return &newStats +} diff --git a/v2/bolt_store.go b/v2/bolt_store.go index 42c947b..14bafb7 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -6,6 +6,7 @@ import ( "os" "time" + metrics "github.com/armon/go-metrics" v1 "github.com/boltdb/bolt" "github.com/hashicorp/raft" "go.etcd.io/bbolt" @@ -152,6 +153,8 @@ func (b *BoltStore) LastIndex() (uint64, error) { // GetLog is used to retrieve a log from Bbolt at a given index. func (b *BoltStore) GetLog(idx uint64, log *raft.Log) error { + defer metrics.MeasureSince([]string{"raft", "boltdb", "getLog"}, time.Now()) + tx, err := b.conn.Begin(false) if err != nil { return err @@ -174,24 +177,33 @@ func (b *BoltStore) StoreLog(log *raft.Log) error { // StoreLogs is used to store a set of raft logs func (b *BoltStore) StoreLogs(logs []*raft.Log) error { + defer metrics.MeasureSince([]string{"raft", "boltdb", "storeLogs"}, time.Now()) tx, err := b.conn.Begin(true) if err != nil { return err } defer tx.Rollback() + batchSize := 0 for _, log := range logs { key := uint64ToBytes(log.Index) val, err := encodeMsgPack(log) if err != nil { return err } + + logLen := val.Len() bucket := tx.Bucket(dbLogs) if err := bucket.Put(key, val.Bytes()); err != nil { return err } + batchSize += logLen + metrics.AddSample([]string{"raft", "boltdb", "logSize"}, float32(logLen)) } + metrics.AddSample([]string{"raft", "boltdb", "logsPerBatch"}, float32(len(logs))) + metrics.AddSample([]string{"raft", "boltdb", "logBatchSize"}, float32(batchSize)) + return tx.Commit() } diff --git a/v2/go.mod b/v2/go.mod index 457ad49..cfa41dc 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/hashicorp/raft-boltdb/v2 go 1.12 require ( + github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 diff --git a/v2/metrics.go b/v2/metrics.go new file mode 100644 index 0000000..1480ff7 --- /dev/null +++ b/v2/metrics.go @@ -0,0 +1,68 @@ +package raftboltdb + +import ( + "context" + "time" + + metrics "github.com/armon/go-metrics" + "go.etcd.io/bbolt" +) + +const ( + defaultMetricsInterval = 5 * time.Second +) + +// RunMetrics should be executed in a go routine and will periodically emit +// metrics on the given interval until the context has been cancelled. +func (b *BoltStore) RunMetrics(ctx context.Context, interval time.Duration) { + if interval == 0 { + interval = defaultMetricsInterval + } + + tick := time.NewTicker(interval) + defer tick.Stop() + + stats := b.emitMetrics(nil) + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + stats = b.emitMetrics(stats) + } + } +} + +func (b *BoltStore) emitMetrics(prev *bbolt.Stats) *bbolt.Stats { + newStats := b.conn.Stats() + + stats := newStats + if prev != nil { + stats = newStats.Sub(prev) + } + + // freelist metrics + metrics.SetGauge([]string{"raft", "boltdb", "numFreePages"}, float32(newStats.FreePageN)) + metrics.SetGauge([]string{"raft", "boltdb", "numPendingPages"}, float32(newStats.PendingPageN)) + metrics.SetGauge([]string{"raft", "boltdb", "freePageBytes"}, float32(newStats.FreeAlloc)) + metrics.SetGauge([]string{"raft", "boltdb", "freelistBytes"}, float32(newStats.FreelistInuse)) + + // txn metrics + metrics.IncrCounter([]string{"raft", "boltdb", "totalReadTxn"}, float32(stats.TxN)) + metrics.SetGauge([]string{"raft", "boltdb", "openReadTxn"}, float32(newStats.OpenTxN)) + + // tx stats + metrics.SetGauge([]string{"raft", "boltdb", "txstats", "pageCount"}, float32(newStats.TxStats.PageCount)) + metrics.SetGauge([]string{"raft", "boltdb", "txstats", "pageAlloc"}, float32(newStats.TxStats.PageAlloc)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "cursorCount"}, float32(stats.TxStats.CursorCount)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "nodeCount"}, float32(stats.TxStats.NodeCount)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "nodeDeref"}, float32(stats.TxStats.NodeDeref)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "rebalance"}, float32(stats.TxStats.Rebalance)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "rebalanceTime"}, float32(stats.TxStats.RebalanceTime.Nanoseconds())/1000000) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "split"}, float32(stats.TxStats.Split)) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "spill"}, float32(stats.TxStats.Spill)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "spillTime"}, float32(stats.TxStats.SpillTime.Nanoseconds())/1000000) + metrics.IncrCounter([]string{"raft", "boltdb", "txstats", "write"}, float32(stats.TxStats.Write)) + metrics.AddSample([]string{"raft", "boltdb", "txstats", "writeTime"}, float32(stats.TxStats.WriteTime.Nanoseconds())/1000000) + return &newStats +}