From 28de9af0bec4c55da68f246231879eb73bbff372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Tue, 19 Jul 2022 15:54:02 +0800 Subject: [PATCH 1/8] feat: add mertics config & exporter --- config/config.go | 29 +++++++------- go.mod | 8 ++-- go.sum | 23 +++++++++++- main.go | 4 +- metrics/exporter.go | 56 +++++++++++++++++++++++++++ metrics/metrics.go | 92 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 metrics/exporter.go create mode 100644 metrics/metrics.go diff --git a/config/config.go b/config/config.go index dc9df54c..55f06139 100644 --- a/config/config.go +++ b/config/config.go @@ -9,16 +9,17 @@ import ( ) type Config struct { - DB DbConfig `toml:"db"` - JWT JWTConfig `toml:"jwt"` - Log LogConfig `toml:"log"` - API APIConfig `toml:"api"` - Node NodeConfig `toml:"node"` - MessageService MessageServiceConfig `toml:"messageService"` - MessageState MessageStateConfig `toml:"messageState"` - Gateway GatewayConfig `toml:"gateway"` - RateLimit RateLimitConfig `toml:"rateLimit"` - Trace metrics.TraceConfig `toml:"tracing"` + DB DbConfig `toml:"db"` + JWT JWTConfig `toml:"jwt"` + Log LogConfig `toml:"log"` + API APIConfig `toml:"api"` + Node NodeConfig `toml:"node"` + MessageService MessageServiceConfig `toml:"messageService"` + MessageState MessageStateConfig `toml:"messageState"` + Gateway GatewayConfig `toml:"gateway"` + RateLimit RateLimitConfig `toml:"rateLimit"` + Trace *metrics.TraceConfig `toml:"tracing"` + Metrics *metrics.MetricsConfig `toml:"metrics"` } type NodeConfig struct { @@ -137,11 +138,7 @@ func DefaultConfig() *Config { Cfg: gatewayTypes.Config{}, }, RateLimit: RateLimitConfig{Redis: ""}, - Trace: metrics.TraceConfig{ - JaegerEndpoint: "", - ProbabilitySampler: 1.0, - JaegerTracingEnabled: false, - ServerName: "venus-messenger", - }, + Trace: metrics.DefaultTraceConfig(), + Metrics: metrics.DefaultMetricsConfig(), } } diff --git a/go.mod b/go.mod index 08f49e3c..94398af8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/filecoin-project/venus-messager go 1.17 require ( + contrib.go.opencensus.io/exporter/graphite v0.0.0-20200424223504-26b90655e0ce // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/fatih/color v1.13.0 github.com/filecoin-project/go-address v0.0.6 @@ -15,7 +16,7 @@ require ( github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/google/uuid v1.3.0 github.com/hunjixin/automapper v0.0.0-20191127090318-9b979ce72ce2 - github.com/ipfs-force-community/metrics v1.0.1-0.20211228055608-9462dc86e157 + github.com/ipfs-force-community/metrics v1.0.1-0.20220719063006-2c54bb379466 github.com/ipfs-force-community/venus-common-utils v0.0.0-20210924063144-1d3a5b30de87 github.com/ipfs-force-community/venus-gateway v1.6.0 github.com/ipfs/go-cid v0.1.0 @@ -36,6 +37,8 @@ require ( modernc.org/mathutil v1.1.1 ) +require go.opencensus.io v0.23.0 + require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect @@ -163,10 +166,7 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect github.com/ugorji/go/codec v1.2.4 // indirect - github.com/warpfork/go-testmark v0.9.0 // indirect github.com/whyrusleeping/go-logging v0.0.1 // indirect - github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 // indirect - go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.12.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/go.sum b/go.sum index 94111617..c977c122 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +contrib.go.opencensus.io/exporter/graphite v0.0.0-20200424223504-26b90655e0ce h1:pIj0fUj3tAZL0CiuN0Qr46HjlTEwBQyms121goB0WLI= +contrib.go.opencensus.io/exporter/graphite v0.0.0-20200424223504-26b90655e0ce/go.mod h1:sniM1YEZcR+VoRPJOiLDegQYhCeGa/7cizeouZvDmf8= contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0= contrib.go.opencensus.io/exporter/prometheus v0.3.0/go.mod h1:rpCPVQKhiyH8oomWgm34ZmgIdZa8OVYO5WAIygPbBBE= @@ -322,6 +324,7 @@ github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88Oq github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.20.1-v16-2/go.mod h1:JLP8bltMbPVhOULcHxE+QFg3b8/a9J8NbcA6Qf69W0k= +github.com/filecoin-project/go-fil-markets v1.20.1/go.mod h1:QV767KIWHrikVK8R0u2wTc5wkee4gXOf5/AfxDoQckw= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -341,6 +344,7 @@ github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= github.com/filecoin-project/go-state-types v0.1.5/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= @@ -348,6 +352,7 @@ github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/ github.com/filecoin-project/go-state-types v0.1.10 h1:YrrJWWh2fU4VPhwHyPlDK5I4mB7bqgnRd3HCm9IOwIU= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statemachine v1.0.2-0.20220322104818-27f8fbb86dfd/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= @@ -356,6 +361,7 @@ github.com/filecoin-project/index-provider v0.5.0/go.mod h1:KHVrP2vU3YuScb+fawOb github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= github.com/filecoin-project/specs-actors v0.9.15-0.20220514164640-94e0d5e123bd/go.mod h1:pjGEe3QlWtK20ju/aFRsiArbMX6Cn8rqEhhsiCM9xYE= github.com/filecoin-project/specs-actors v0.9.15 h1:3VpKP5/KaDUHQKAMOg4s35g/syDaEBueKLws0vbsjMc= github.com/filecoin-project/specs-actors v0.9.15/go.mod h1:pjGEe3QlWtK20ju/aFRsiArbMX6Cn8rqEhhsiCM9xYE= @@ -363,9 +369,11 @@ github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb github.com/filecoin-project/specs-actors/v2 v2.3.6 h1:UxnWTfQd7JsOae39/aHCK0m1IBjdcyymCJfqxuSkn+g= github.com/filecoin-project/specs-actors/v2 v2.3.6/go.mod h1:DJMpxVRXvev9t8P0XWA26RmTzN+MHiL9IlItVLT0zUc= github.com/filecoin-project/specs-actors/v3 v3.1.0/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww= +github.com/filecoin-project/specs-actors/v3 v3.1.1/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww= github.com/filecoin-project/specs-actors/v3 v3.1.2 h1:Gq3gAbvdGLA/D0GKz1IJfewt9Fh7gA32TPt46Xv+1Cw= github.com/filecoin-project/specs-actors/v3 v3.1.2/go.mod h1:uOJn+m6W8OW/1mdWMEvxeM1cjQPxmps7s1Z4bJ9V4kY= github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= +github.com/filecoin-project/specs-actors/v4 v4.0.1/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= github.com/filecoin-project/specs-actors/v4 v4.0.2 h1:VTsv30kIf1Keo8Jlu6Omco+2Ud0pG4EN5UAzyYCibh8= github.com/filecoin-project/specs-actors/v4 v4.0.2/go.mod h1:zT0GVFxwFS93prGK0b/rMd1sePjRQKfAuodQ9DFAd6Y= github.com/filecoin-project/specs-actors/v5 v5.0.4/go.mod h1:5BAKRAMsOOlD8+qCw4UvT/lTLInCJ3JwOWZbX8Ipwq4= @@ -381,13 +389,16 @@ github.com/filecoin-project/specs-actors/v7 v7.0.1 h1:w72xCxijK7xs1qzmJiw+WYJaVt github.com/filecoin-project/specs-actors/v7 v7.0.1/go.mod h1:tPLEYXoXhcpyLh69Ccq91SOuLXsPWjHiY27CzawjUEk= github.com/filecoin-project/specs-actors/v8 v8.0.1 h1:4u0tIRJeT5G7F05lwLRIsDnsrN+bJ5Ixj6h49Q7uE2Y= github.com/filecoin-project/specs-actors/v8 v8.0.1/go.mod h1:UYIPg65iPWoFw5NEftREdJwv9b/5yaLKdCgTvNI/2FA= +github.com/filecoin-project/specs-storage v0.2.2/go.mod h1:6cc/lncmAxMUocPi0z1EPCX63beIX7F7UnlmUZ3hLQo= github.com/filecoin-project/specs-storage v0.4.1 h1:yvLEaLZj8f+uByhNC4mFOtCUyL2wQku+NGBp6hjTe9M= github.com/filecoin-project/specs-storage v0.4.1/go.mod h1:Z2eK6uMwAOSLjek6+sy0jNV2DSsMEENziMUz0GHRFBw= github.com/filecoin-project/storetheindex v0.3.5/go.mod h1:0r3d0kSpK63O6AvLr1CjAINLi+nWD49clzcnKV+GLpI= github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= +github.com/filecoin-project/venus v1.2.4/go.mod h1:hJULXHGAnWuq5S5KRtPkwbT8DqgM9II7NwyNU7t59D0= github.com/filecoin-project/venus v1.6.0/go.mod h1:ukA+xwqDs40lixoa+HDNfuN8b1G4jpm4k0ujceVejSk= github.com/filecoin-project/venus v1.6.1-0.20220718091042-a51da69e1731 h1:DfWH8MQ75AWe1856rgP5eN0bwJCD2bDySJnuG6lOHEM= github.com/filecoin-project/venus v1.6.1-0.20220718091042-a51da69e1731/go.mod h1:ukA+xwqDs40lixoa+HDNfuN8b1G4jpm4k0ujceVejSk= +github.com/filecoin-project/venus-auth v1.3.2/go.mod h1:m5Jog2GYxztwP7w3m/iJdv/V1/bTcAVU9rm/CbhxRQU= github.com/filecoin-project/venus-auth v1.6.0 h1:DLl7q5g1eh6UTpp98MLpRWAI79k6TUw1Myh/RLeaFpU= github.com/filecoin-project/venus-auth v1.6.0/go.mod h1:x/Cv3zz9z5O+/uqIKgYtk5UsL7nYu+CtiPjyVQ8Lywg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -472,6 +483,7 @@ github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -688,8 +700,9 @@ github.com/ipfs-force-community/go-jsonrpc v0.1.4-0.20211201033628-fc1430d095f6 github.com/ipfs-force-community/go-jsonrpc v0.1.4-0.20211201033628-fc1430d095f6/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= github.com/ipfs-force-community/metrics v1.0.0/go.mod h1:mn40SioMuKtjmRumHFy/fJ26Pn028XuDjUJE9dorjyw= github.com/ipfs-force-community/metrics v1.0.1-0.20211022060227-11142a08b729/go.mod h1:mn40SioMuKtjmRumHFy/fJ26Pn028XuDjUJE9dorjyw= -github.com/ipfs-force-community/metrics v1.0.1-0.20211228055608-9462dc86e157 h1:HkFCM7oueZC6Lnj6HyJC6gxC0qbjBlWCYg36x3omKyk= github.com/ipfs-force-community/metrics v1.0.1-0.20211228055608-9462dc86e157/go.mod h1:mn40SioMuKtjmRumHFy/fJ26Pn028XuDjUJE9dorjyw= +github.com/ipfs-force-community/metrics v1.0.1-0.20220719063006-2c54bb379466 h1:oPQtpBVmYot48fGuUwqVuzD1ilMnHUXkkGSWy5VzUL8= +github.com/ipfs-force-community/metrics v1.0.1-0.20220719063006-2c54bb379466/go.mod h1:MOGtsAHKkSXi/ce9Ggci7nXq5jShePzwPDvdBSGDETk= github.com/ipfs-force-community/venus-common-utils v0.0.0-20210924063144-1d3a5b30de87 h1:1PTFrWfxN5b8y1Cnw403PZ9Gy5RLHN4FXmUM4nTt9Yg= github.com/ipfs-force-community/venus-common-utils v0.0.0-20210924063144-1d3a5b30de87/go.mod h1:RTVEOzM+hkpqmcEWpyLDkx1oGO5r9ZWCgYxG/CsXzJQ= github.com/ipfs-force-community/venus-gateway v1.6.0 h1:uFx01GI78ZhpfgIBXsoa0hK1p0zuARuK69weQ9ZHkZU= @@ -1429,6 +1442,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= @@ -1922,6 +1936,7 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325/go.mod h1:g7ckxrjiFh8mi1AY7ox23PZD0g6QU/TxW3U3unX7I3A= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -1930,8 +1945,9 @@ github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829 h1:wb7xrDzfkLgPHsSEB github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= -github.com/xorcare/golden v0.6.0 h1:E8emU8bhyMIEpYmgekkTUaw4vtcrRE+Wa0c5wYIcgXc= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZDpfMfWg7xk29yEOZiXmo/wZl+utTI8= +github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2193,6 +2209,7 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= @@ -2350,6 +2367,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2593,6 +2611,7 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= diff --git a/main.go b/main.go index 2e2d2d69..0f305831 100644 --- a/main.go +++ b/main.go @@ -235,7 +235,8 @@ func runAction(ctx *cli.Context) error { provider := fx.Options( fx.Logger(fxLogger{log}), // prover - fx.Supply(cfg, &cfg.DB, &cfg.API, &cfg.JWT, &cfg.Node, &cfg.Log, &cfg.MessageService, &cfg.MessageState, &cfg.Gateway, &cfg.RateLimit, &cfg.Trace), + fx.Supply(cfg, &cfg.DB, &cfg.API, &cfg.JWT, &cfg.Node, &cfg.Log, &cfg.MessageService, + &cfg.MessageState, &cfg.Gateway, &cfg.RateLimit, cfg.Trace, cfg.Metrics), fx.Supply(log), fx.Supply(client), fx.Supply(walletClient), @@ -269,6 +270,7 @@ func runAction(ctx *cli.Context) error { fx.Invoke(models.AutoMigrate), fx.Invoke(service.StartNodeEvents), fx.Invoke(metrics.SetupJaeger), + fx.Invoke(metrics.SetupMetrics), ) apiOption := fx.Options( diff --git a/metrics/exporter.go b/metrics/exporter.go new file mode 100644 index 00000000..05b21321 --- /dev/null +++ b/metrics/exporter.go @@ -0,0 +1,56 @@ +package metrics + +import ( + "context" + + "go.opencensus.io/stats/view" + "go.uber.org/fx" + + "github.com/ipfs-force-community/metrics" + + "github.com/filecoin-project/venus-messager/log" +) + +func SetupMetrics(lc fx.Lifecycle, metricsConfig *metrics.MetricsConfig, log *log.Logger) error { + log.Infof("metrics config: enabled: %v, exporter type: %s, prometheus: %v, graphite: %v\n", + metricsConfig.Enabled, metricsConfig.Exporter.Type, *metricsConfig.Exporter.Prometheus, + *metricsConfig.Exporter.Graphite) + + if !metricsConfig.Enabled { + return nil + } + + if err := view.Register( + MessagerNodeViews..., + ); err != nil { + log.Fatalf("Cannot register the view: %v", err) + } + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + switch metricsConfig.Exporter.Type { + case metrics.ETPrometheus: + go func() { + if err := metrics.RegisterPrometheusExporter(ctx, metricsConfig.Exporter.Prometheus); err != nil { + log.Errorf("Register prometheus exporter err: %v", err) + } + log.Info("Prometheus exporter server graceful shutdown successful") + }() + + case metrics.ETGraphite: + if err := metrics.RegisterGraphiteExporter(ctx, metricsConfig.Exporter.Graphite); err != nil { + log.Errorf("failed to register the exporter: %v", err) + } + default: + log.Warnf("invalid exporter type: %s", metricsConfig.Exporter.Type) + } + + return nil + }, + OnStop: func(ctx context.Context) error { + return nil + }, + }) + + return nil +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 00000000..56bc2d76 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,92 @@ +package metrics + +import ( + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +// Global Tags +var ( + WalletAddress, _ = tag.NewKey("wallet") +) + +var ( + WalletBalance = stats.Int64("wallet_balance", "Wallet balance", stats.UnitDimensionless) + WalletDBNonce = stats.Int64("wallet_db_nonce", "Wallet nonce in db", stats.UnitDimensionless) + WalletChainNonce = stats.Int64("wallet_chain_nonce", "Wallet nonce on the chain", stats.UnitDimensionless) + + NumOfUnFillMsg = stats.Int64("num_of_unfill_msg", "The number of unFill msg", stats.UnitDimensionless) + NumOfFillMsg = stats.Int64("num_of_fill_msg", "The number of fill Msg", stats.UnitDimensionless) + NumOfFailedMsg = stats.Int64("num_of_failed_msg", "The number of failed msg", stats.UnitDimensionless) + NumOfReplacedMsg = stats.Int64("num_of_replace_msg", "The number of replaced msg", stats.UnitDimensionless) + + NumOfMsgBlockedThreeMinutes = stats.Int64("blocked_three_minutes_msgs", "Number of messages blocked for more than 3 minutes", stats.UnitDimensionless) + NumOfMsgBlockedFiveMinutes = stats.Int64("blocked_five_minutes_msgs", "Number of messages blocked for more than 5 minutes", stats.UnitDimensionless) +) + +var ( + WalletBalanceView = &view.View{ + Measure: WalletBalance, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + WalletChainNonceView = &view.View{ + Measure: WalletChainNonce, + Aggregation: view.Count(), + TagKeys: []tag.Key{WalletAddress}, + } + WalletDBNonceView = &view.View{ + Measure: WalletDBNonce, + Aggregation: view.Count(), + TagKeys: []tag.Key{WalletAddress}, + } + + NumOfUnFillMsgView = &view.View{ + Measure: NumOfUnFillMsg, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + NumOfFillMsgView = &view.View{ + Measure: NumOfFillMsg, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + NumOfFailedMsgView = &view.View{ + Measure: NumOfFailedMsg, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + NumOfReplacedMsgView = &view.View{ + Measure: NumOfReplacedMsg, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + + NumOfMsgBlockedThreeMinutesView = &view.View{ + Measure: NumOfMsgBlockedThreeMinutes, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } + NumOfMsgBlockedFiveMinutesView = &view.View{ + Measure: NumOfMsgBlockedFiveMinutes, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{WalletAddress}, + } +) + +var MessagerNodeViews = []*view.View{ + WalletBalanceView, + WalletChainNonceView, + WalletDBNonceView, + + NumOfUnFillMsgView, + NumOfFillMsgView, + NumOfFailedMsgView, + NumOfReplacedMsgView, + + NumOfMsgBlockedThreeMinutesView, + NumOfMsgBlockedFiveMinutesView, +} From 740d436fd31a4e859e6acbcbff2ea5d1b69b4dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Tue, 19 Jul 2022 16:11:05 +0800 Subject: [PATCH 2/8] feat: record address-related metrics --- metrics/metrics.go | 2 -- service/address_service.go | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 56bc2d76..11637ee4 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,8 +1,6 @@ package metrics import ( - "time" - "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" diff --git a/service/address_service.go b/service/address_service.go index 4a737655..17419a32 100644 --- a/service/address_service.go +++ b/service/address_service.go @@ -4,12 +4,19 @@ import ( "context" "errors" "fmt" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/tag" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/venus-messager/gateway" "github.com/filecoin-project/venus-messager/log" + "github.com/filecoin-project/venus-messager/metrics" "github.com/filecoin-project/venus-messager/models/repo" + v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" venusTypes "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/messager" @@ -40,6 +47,9 @@ func NewAddressService(repo repo.Repo, walletClient: walletClient, } + // 本身缺少 global context + go addressService.recordMetricsProc(context.TODO()) + return addressService } @@ -165,3 +175,37 @@ func (addressService *AddressService) ActiveAddresses() map[address.Address]stru return addrs } + +func (addressService *AddressService) recordMetricsProc(ctx context.Context) { + tm := time.NewTicker(time.Second * 60) + defer tm.Stop() + + for { + select { + case <-ctx.Done(): + addressService.log.Info("Stop address record metrics") + return + case <-tm.C: + addrs, err := addressService.ListActiveAddress(ctx) + if err != nil { + addressService.log.Errorf("get address list err: %s", err) + } + + for _, addr := range addrs { + + ctx, _ = tag.New( + ctx, + tag.Upsert(metrics.WalletAddress, addr.Addr.String()), + ) + stats.Record(ctx, metrics.WalletDBNonce.M(int64(addr.Nonce))) + + actor, err := addressService.nodeClient.StateGetActor(ctx, addr.Addr, venusTypes.EmptyTSK) + if err != nil { + addressService.log.Errorf("get actor err: %s", err) + } + stats.Record(ctx, metrics.WalletBalance.M(actor.Balance.Int64())) + stats.Record(ctx, metrics.WalletChainNonce.M(int64(actor.Nonce))) + } + } + } +} From 8466c0431939c68f12edb97aaee502a72907e4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Tue, 19 Jul 2022 17:28:48 +0800 Subject: [PATCH 3/8] feat: record num of msgs for the specified status metrics --- metrics/metrics.go | 14 ++---- service/address_service.go | 42 ------------------ service/message_service.go | 88 +++++++++++++++++++++++++++++++++++--- 3 files changed, 86 insertions(+), 58 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 11637ee4..6afe8f26 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -16,10 +16,9 @@ var ( WalletDBNonce = stats.Int64("wallet_db_nonce", "Wallet nonce in db", stats.UnitDimensionless) WalletChainNonce = stats.Int64("wallet_chain_nonce", "Wallet nonce on the chain", stats.UnitDimensionless) - NumOfUnFillMsg = stats.Int64("num_of_unfill_msg", "The number of unFill msg", stats.UnitDimensionless) - NumOfFillMsg = stats.Int64("num_of_fill_msg", "The number of fill Msg", stats.UnitDimensionless) - NumOfFailedMsg = stats.Int64("num_of_failed_msg", "The number of failed msg", stats.UnitDimensionless) - NumOfReplacedMsg = stats.Int64("num_of_replace_msg", "The number of replaced msg", stats.UnitDimensionless) + NumOfUnFillMsg = stats.Int64("num_of_unfill_msg", "The number of unFill msg", stats.UnitDimensionless) + NumOfFillMsg = stats.Int64("num_of_fill_msg", "The number of fill Msg", stats.UnitDimensionless) + NumOfFailedMsg = stats.Int64("num_of_failed_msg", "The number of failed msg", stats.UnitDimensionless) NumOfMsgBlockedThreeMinutes = stats.Int64("blocked_three_minutes_msgs", "Number of messages blocked for more than 3 minutes", stats.UnitDimensionless) NumOfMsgBlockedFiveMinutes = stats.Int64("blocked_five_minutes_msgs", "Number of messages blocked for more than 5 minutes", stats.UnitDimensionless) @@ -55,12 +54,6 @@ var ( NumOfFailedMsgView = &view.View{ Measure: NumOfFailedMsg, Aggregation: view.LastValue(), - TagKeys: []tag.Key{WalletAddress}, - } - NumOfReplacedMsgView = &view.View{ - Measure: NumOfReplacedMsg, - Aggregation: view.LastValue(), - TagKeys: []tag.Key{WalletAddress}, } NumOfMsgBlockedThreeMinutesView = &view.View{ @@ -83,7 +76,6 @@ var MessagerNodeViews = []*view.View{ NumOfUnFillMsgView, NumOfFillMsgView, NumOfFailedMsgView, - NumOfReplacedMsgView, NumOfMsgBlockedThreeMinutesView, NumOfMsgBlockedFiveMinutesView, diff --git a/service/address_service.go b/service/address_service.go index 17419a32..e681a701 100644 --- a/service/address_service.go +++ b/service/address_service.go @@ -4,17 +4,12 @@ import ( "context" "errors" "fmt" - "time" - - "go.opencensus.io/stats" - "go.opencensus.io/tag" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/venus-messager/gateway" "github.com/filecoin-project/venus-messager/log" - "github.com/filecoin-project/venus-messager/metrics" "github.com/filecoin-project/venus-messager/models/repo" v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" @@ -47,9 +42,6 @@ func NewAddressService(repo repo.Repo, walletClient: walletClient, } - // 本身缺少 global context - go addressService.recordMetricsProc(context.TODO()) - return addressService } @@ -175,37 +167,3 @@ func (addressService *AddressService) ActiveAddresses() map[address.Address]stru return addrs } - -func (addressService *AddressService) recordMetricsProc(ctx context.Context) { - tm := time.NewTicker(time.Second * 60) - defer tm.Stop() - - for { - select { - case <-ctx.Done(): - addressService.log.Info("Stop address record metrics") - return - case <-tm.C: - addrs, err := addressService.ListActiveAddress(ctx) - if err != nil { - addressService.log.Errorf("get address list err: %s", err) - } - - for _, addr := range addrs { - - ctx, _ = tag.New( - ctx, - tag.Upsert(metrics.WalletAddress, addr.Addr.String()), - ) - stats.Record(ctx, metrics.WalletDBNonce.M(int64(addr.Nonce))) - - actor, err := addressService.nodeClient.StateGetActor(ctx, addr.Addr, venusTypes.EmptyTSK) - if err != nil { - addressService.log.Errorf("get actor err: %s", err) - } - stats.Record(ctx, metrics.WalletBalance.M(actor.Balance.Int64())) - stats.Record(ctx, metrics.WalletChainNonce.M(int64(actor.Nonce))) - } - } - } -} diff --git a/service/message_service.go b/service/message_service.go index 6c1ce995..ba72373e 100644 --- a/service/message_service.go +++ b/service/message_service.go @@ -8,20 +8,26 @@ import ( "strings" "time" + "github.com/ipfs/go-cid" + "gorm.io/gorm" + + "go.opencensus.io/stats" + "go.opencensus.io/tag" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/venus/pkg/constants" - v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" - venusTypes "github.com/filecoin-project/venus/venus-shared/types" - "github.com/ipfs/go-cid" - "gorm.io/gorm" "github.com/filecoin-project/venus-messager/filestore" "github.com/filecoin-project/venus-messager/gateway" "github.com/filecoin-project/venus-messager/log" + "github.com/filecoin-project/venus-messager/metrics" "github.com/filecoin-project/venus-messager/models/repo" + + "github.com/filecoin-project/venus/pkg/constants" + v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" + venusTypes "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/messager" ) @@ -103,6 +109,9 @@ func NewMessageService(repo repo.Repo, ms.log.Infof("load tipset file failed: %v", err) } + // 本身缺少 global context + go ms.recordMetricsProc(context.TODO()) + return ms, ms.verifyNetworkName() } @@ -1034,6 +1043,75 @@ func (ms *MessageService) ClearUnFillMessage(ctx context.Context, addr address.A } } +func (ms *MessageService) recordMetricsProc(ctx context.Context) { + tm := time.NewTicker(time.Second * 60) + defer tm.Stop() + + for { + select { + case <-ctx.Done(): + ms.log.Info("Stop record metrics") + return + case <-tm.C: + addrs, err := ms.addressService.ListActiveAddress(ctx) + if err != nil { + ms.addressService.log.Errorf("get address list err: %s", err) + } + + for _, addr := range addrs { + ctx, _ = tag.New( + ctx, + tag.Upsert(metrics.WalletAddress, addr.Addr.String()), + ) + stats.Record(ctx, metrics.WalletDBNonce.M(int64(addr.Nonce))) + + actor, err := ms.addressService.nodeClient.StateGetActor(ctx, addr.Addr, venusTypes.EmptyTSK) + if err != nil { + ms.addressService.log.Errorf("get actor err: %s", err) + } else { + stats.Record(ctx, metrics.WalletBalance.M(actor.Balance.Int64())) + stats.Record(ctx, metrics.WalletChainNonce.M(int64(actor.Nonce))) + } + + msgs, err := ms.repo.MessageRepo().ListUnFilledMessage(addr.Addr) + if err != nil { + ms.addressService.log.Errorf("get unFilled msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfUnFillMsg.M(int64(len(msgs)))) + } + + msgs, err = ms.repo.MessageRepo().ListFilledMessageByAddress(addr.Addr) + if err != nil { + ms.addressService.log.Errorf("get filled msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfFillMsg.M(int64(len(msgs)))) + } + + msgs, err = ms.repo.MessageRepo().ListFailedMessage() + if err != nil { + ms.addressService.log.Errorf("get failed msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfFailedMsg.M(int64(len(msgs)))) + } + + msgs, err = ms.repo.MessageRepo().ListBlockedMessage(addr.Addr, 3*time.Minute) + if err != nil { + ms.addressService.log.Errorf("get blocked msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfMsgBlockedThreeMinutes.M(int64(len(msgs)))) + } + + msgs, err = ms.repo.MessageRepo().ListBlockedMessage(addr.Addr, 5*time.Minute) + if err != nil { + ms.addressService.log.Errorf("get blocked msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfMsgBlockedFiveMinutes.M(int64(len(msgs)))) + } + } + } + } +} + const ( ReplaceByFeeRatioDefault = 1.25 RbfDenom = 256 From e0ca54063ac2a4f584cbd1274f9df66aec39920c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Wed, 20 Jul 2022 10:38:24 +0800 Subject: [PATCH 4/8] feat: record chain head stable & each round of selection msgs metrics --- metrics/metrics.go | 36 ++++++++++++++++++++++++++++++++ service/message_service.go | 23 ++++++++++++-------- service/message_state_refresh.go | 10 ++++++++- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 6afe8f26..347c3d37 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -22,6 +22,13 @@ var ( NumOfMsgBlockedThreeMinutes = stats.Int64("blocked_three_minutes_msgs", "Number of messages blocked for more than 3 minutes", stats.UnitDimensionless) NumOfMsgBlockedFiveMinutes = stats.Int64("blocked_five_minutes_msgs", "Number of messages blocked for more than 5 minutes", stats.UnitDimensionless) + + SelectedMsgNumOfLastRound = stats.Int64("selected_msg_num", "Number of selected messages in the last round", stats.UnitDimensionless) + ToPushMsgNumOfLastRound = stats.Int64("topush_msg_num", "Number of to-push messages in the last round", stats.UnitDimensionless) + ExpiredMsgNumOfLastRound = stats.Int64("expired_msg_num", "Number of expired messages in the last round", stats.UnitDimensionless) + ErrMsgNumOfLastRound = stats.Int64("err_msg_num", "Number of err messages in the last round", stats.UnitDimensionless) + + ChainHeadStableDelay = stats.Int64("chain_head_stable_s", "Duration of chain head stabilization", stats.UnitSeconds) ) var ( @@ -66,6 +73,28 @@ var ( Aggregation: view.LastValue(), TagKeys: []tag.Key{WalletAddress}, } + + SelectedMsgNumOfLastRoundView = &view.View{ + Measure: SelectedMsgNumOfLastRound, + Aggregation: view.LastValue(), + } + ToPushMsgNumOfLastRoundView = &view.View{ + Measure: ToPushMsgNumOfLastRound, + Aggregation: view.LastValue(), + } + ExpiredMsgNumOfLastRoundView = &view.View{ + Measure: ExpiredMsgNumOfLastRound, + Aggregation: view.LastValue(), + } + ErrMsgNumOfLastRoundView = &view.View{ + Measure: ErrMsgNumOfLastRound, + Aggregation: view.LastValue(), + } + + ChainHeadStableDelayView = &view.View{ + Measure: ChainHeadStableDelay, + Aggregation: view.LastValue(), + } ) var MessagerNodeViews = []*view.View{ @@ -79,4 +108,11 @@ var MessagerNodeViews = []*view.View{ NumOfMsgBlockedThreeMinutesView, NumOfMsgBlockedFiveMinutesView, + + SelectedMsgNumOfLastRoundView, + ToPushMsgNumOfLastRoundView, + ExpiredMsgNumOfLastRoundView, + ErrMsgNumOfLastRoundView, + + ChainHeadStableDelayView, } diff --git a/service/message_service.go b/service/message_service.go index ba72373e..8e19ff7c 100644 --- a/service/message_service.go +++ b/service/message_service.go @@ -554,6 +554,11 @@ func (ms *MessageService) pushMessageToPool(ctx context.Context, ts *venusTypes. return err } ms.log.Infof("current loop select result | SelectMsg: %d | ExpireMsg: %d | ToPushMsg: %d | ErrMsg: %d", len(selectResult.SelectMsg), len(selectResult.ExpireMsg), len(selectResult.ToPushMsg), len(selectResult.ErrMsg)) + stats.Record(ctx, metrics.SelectedMsgNumOfLastRound.M(int64(len(selectResult.SelectMsg)))) + stats.Record(ctx, metrics.ToPushMsgNumOfLastRound.M(int64(len(selectResult.ToPushMsg)))) + stats.Record(ctx, metrics.ExpiredMsgNumOfLastRound.M(int64(len(selectResult.ExpireMsg)))) + stats.Record(ctx, metrics.ErrMsgNumOfLastRound.M(int64(len(selectResult.ErrMsg)))) + tSaveDb := time.Now() ms.log.Infof("start to save to database") //save to db @@ -1087,27 +1092,27 @@ func (ms *MessageService) recordMetricsProc(ctx context.Context) { stats.Record(ctx, metrics.NumOfFillMsg.M(int64(len(msgs)))) } - msgs, err = ms.repo.MessageRepo().ListFailedMessage() - if err != nil { - ms.addressService.log.Errorf("get failed msg err: %s", err) - } else { - stats.Record(ctx, metrics.NumOfFailedMsg.M(int64(len(msgs)))) - } - msgs, err = ms.repo.MessageRepo().ListBlockedMessage(addr.Addr, 3*time.Minute) if err != nil { - ms.addressService.log.Errorf("get blocked msg err: %s", err) + ms.addressService.log.Errorf("get blocked three minutes msg err: %s", err) } else { stats.Record(ctx, metrics.NumOfMsgBlockedThreeMinutes.M(int64(len(msgs)))) } msgs, err = ms.repo.MessageRepo().ListBlockedMessage(addr.Addr, 5*time.Minute) if err != nil { - ms.addressService.log.Errorf("get blocked msg err: %s", err) + ms.addressService.log.Errorf("get blocked five minutes msg err: %s", err) } else { stats.Record(ctx, metrics.NumOfMsgBlockedFiveMinutes.M(int64(len(msgs)))) } } + + msgs, err := ms.repo.MessageRepo().ListFailedMessage() + if err != nil { + ms.addressService.log.Errorf("get failed msg err: %s", err) + } else { + stats.Record(ctx, metrics.NumOfFailedMsg.M(int64(len(msgs)))) + } } } } diff --git a/service/message_state_refresh.go b/service/message_state_refresh.go index f99daddb..afd0a84d 100644 --- a/service/message_state_refresh.go +++ b/service/message_state_refresh.go @@ -6,10 +6,13 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" - venustypes "github.com/filecoin-project/venus/venus-shared/types" "github.com/ipfs/go-cid" + "go.opencensus.io/stats" + "github.com/filecoin-project/venus-messager/metrics" "github.com/filecoin-project/venus-messager/models/repo" + + venustypes "github.com/filecoin-project/venus/venus-shared/types" types "github.com/filecoin-project/venus/venus-shared/types/messager" ) @@ -162,6 +165,11 @@ func (ms *MessageService) updateMessageState(ctx context.Context, tsKeys map[abi // delayTrigger wait for stable ts func (ms *MessageService) delayTrigger(ctx context.Context, ts *venustypes.TipSet) { + defer func() { + ds := time.Now().Unix() - int64(ts.MinTimestamp()) + stats.Record(ctx, metrics.ChainHeadStableDelay.M(ds)) + }() + select { case <-time.After(ms.fsRepo.Config().MessageService.WaitingChainHeadStableDuration): ms.triggerPush <- ts From a16de115fafd136bc2a927bfaf92ed3cd2146704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Wed, 20 Jul 2022 10:57:58 +0800 Subject: [PATCH 5/8] fix(metrics): accurate record waiting chainHead stable delay --- metrics/metrics.go | 2 +- service/message_state_refresh.go | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 347c3d37..7f54f5ca 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -28,7 +28,7 @@ var ( ExpiredMsgNumOfLastRound = stats.Int64("expired_msg_num", "Number of expired messages in the last round", stats.UnitDimensionless) ErrMsgNumOfLastRound = stats.Int64("err_msg_num", "Number of err messages in the last round", stats.UnitDimensionless) - ChainHeadStableDelay = stats.Int64("chain_head_stable_s", "Duration of chain head stabilization", stats.UnitSeconds) + ChainHeadStableDelay = stats.Int64("chain_head_stable_s", "Duration of chain head stabilization", stats.UnitSeconds) ) var ( diff --git a/service/message_state_refresh.go b/service/message_state_refresh.go index afd0a84d..4c7ff916 100644 --- a/service/message_state_refresh.go +++ b/service/message_state_refresh.go @@ -165,13 +165,10 @@ func (ms *MessageService) updateMessageState(ctx context.Context, tsKeys map[abi // delayTrigger wait for stable ts func (ms *MessageService) delayTrigger(ctx context.Context, ts *venustypes.TipSet) { - defer func() { - ds := time.Now().Unix() - int64(ts.MinTimestamp()) - stats.Record(ctx, metrics.ChainHeadStableDelay.M(ds)) - }() - select { case <-time.After(ms.fsRepo.Config().MessageService.WaitingChainHeadStableDuration): + ds := time.Now().Unix() - int64(ts.MinTimestamp()) + stats.Record(ctx, metrics.ChainHeadStableDelay.M(ds)) ms.triggerPush <- ts return case <-ctx.Done(): From 94891a372fd3f5ef992730031b70f42baaae56b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Wed, 20 Jul 2022 13:12:15 +0800 Subject: [PATCH 6/8] feat: add histograms metrics for chainHead stable --- metrics/metrics.go | 11 ++++++++++- service/message_state_refresh.go | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index 7f54f5ca..2c21208d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,6 +11,9 @@ var ( WalletAddress, _ = tag.NewKey("wallet") ) +// Distribution +var defaultSecondsDistribution = view.Distribution(8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 60) + var ( WalletBalance = stats.Int64("wallet_balance", "Wallet balance", stats.UnitDimensionless) WalletDBNonce = stats.Int64("wallet_db_nonce", "Wallet nonce in db", stats.UnitDimensionless) @@ -28,7 +31,8 @@ var ( ExpiredMsgNumOfLastRound = stats.Int64("expired_msg_num", "Number of expired messages in the last round", stats.UnitDimensionless) ErrMsgNumOfLastRound = stats.Int64("err_msg_num", "Number of err messages in the last round", stats.UnitDimensionless) - ChainHeadStableDelay = stats.Int64("chain_head_stable_s", "Duration of chain head stabilization", stats.UnitSeconds) + ChainHeadStableDelay = stats.Int64("chain_head_stable_s", "Delay of chain head stabilization", stats.UnitSeconds) + ChainHeadStableDuration = stats.Int64("chain_head_stable_dur_s", "Duration of chain head stabilization", stats.UnitSeconds) ) var ( @@ -95,6 +99,10 @@ var ( Measure: ChainHeadStableDelay, Aggregation: view.LastValue(), } + ChainHeadStableDurationView = &view.View{ + Measure: ChainHeadStableDuration, + Aggregation: defaultSecondsDistribution, + } ) var MessagerNodeViews = []*view.View{ @@ -115,4 +123,5 @@ var MessagerNodeViews = []*view.View{ ErrMsgNumOfLastRoundView, ChainHeadStableDelayView, + ChainHeadStableDurationView, } diff --git a/service/message_state_refresh.go b/service/message_state_refresh.go index 4c7ff916..342aaba9 100644 --- a/service/message_state_refresh.go +++ b/service/message_state_refresh.go @@ -169,6 +169,7 @@ func (ms *MessageService) delayTrigger(ctx context.Context, ts *venustypes.TipSe case <-time.After(ms.fsRepo.Config().MessageService.WaitingChainHeadStableDuration): ds := time.Now().Unix() - int64(ts.MinTimestamp()) stats.Record(ctx, metrics.ChainHeadStableDelay.M(ds)) + stats.Record(ctx, metrics.ChainHeadStableDuration.M(ds)) ms.triggerPush <- ts return case <-ctx.Done(): From 822cbc45473b3c1f3ef7a2d72ee7b073e7f10eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Wed, 20 Jul 2022 13:49:51 +0800 Subject: [PATCH 7/8] docs: add metrics config & use desc --- docs/zh/metrics-config-desc.md | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 docs/zh/metrics-config-desc.md diff --git a/docs/zh/metrics-config-desc.md b/docs/zh/metrics-config-desc.md new file mode 100644 index 00000000..44c64faa --- /dev/null +++ b/docs/zh/metrics-config-desc.md @@ -0,0 +1,69 @@ +# metrics 配置及使用说明 + +## 配置 + +`Metrics` 基本的配置样例如下： +```toml +[Metrics] + # 是否开启metrics指标统计，默认为false + Enabled = false + + [Metrics.Exporter] + # 指标导出器类型，目前可选：prometheus或graphite，默认为prometheus + Type = "prometheus" + + [Metrics.Exporter.Prometheus] + # multiaddr + EndPoint = "/ip4/0.0.0.0/tcp/4568" + # 命名规范: "a_b_c", 不能带"-" + Namespace = "messager01" + # 指标注册表类型，可选：default（默认，会附带程序运行的环境指标）或 define（自定义） + RegistryType = "define" + # prometheus 服务路径 + Path = "/debug/metrics" + # 上报周期，单位为 秒（s） + ReportingPeriod = 10 + + [Metrics.Exporter.Graphite] + # 命名规范: "a_b_c", 不能带"-" + Namespace = "messager01" + # graphite exporter 收集器服务地址 + Host = "127.0.0.1" + # graphite exporter 收集器服务监听端口 + Port = 4568 + # 上报周期，单位为 秒（s） + ReportingPeriod = 10 +``` +## 导出器 + +目前可以选择两类导出器（`exporter`）：`Prometheus exporter` 或 `Graphite exporter`，默认是前者。 + +如果配置 `Prometheus exporter`，则在 `venus-messager` 服务启动时会附带启动 `Prometheus exporter` 的监听服务，可以通过以下方式快速查看指标： + +```bash + $ curl http://192.168.200.107:4568/debug/metrics + # HELP messager01_chain_head_stable_dur_s Duration of chain head stabilization + # TYPE messager01_chain_head_stable_dur_s histogram + messager01_chain_head_stable_dur_s_bucket{le="8"} 0 + messager01_chain_head_stable_dur_s_bucket{le="9"} 11 + messager01_chain_head_stable_dur_s_bucket{le="10"} 27 + messager01_chain_head_stable_dur_s_bucket{le="12"} 43 + messager01_chain_head_stable_dur_s_bucket{le="14"} 48 + messager01_chain_head_stable_dur_s_bucket{le="16"} 49 + messager01_chain_head_stable_dur_s_bucket{le="18"} 49 + messager01_chain_head_stable_dur_s_bucket{le="20"} 49 + messager01_chain_head_stable_dur_s_bucket{le="25"} 49 + messager01_chain_head_stable_dur_s_bucket{le="30"} 49 + messager01_chain_head_stable_dur_s_bucket{le="60"} 50 + messager01_chain_head_stable_dur_s_bucket{le="+Inf"} 50 + messager01_chain_head_stable_dur_s_sum 503.99999999999983 + messager01_chain_head_stable_dur_s_count 50 + # HELP messager01_chain_head_stable_s Delay of chain head stabilization + # TYPE messager01_chain_head_stable_s gauge + messager01_chain_head_stable_s 9 + ... ... +``` + +如果配置 `Graphite exporter`，需要先启动 `Graphite exporter` 的收集器服务， `venus-messager` 服务启动时将指标上报给收集器。服务启动参考 [Graphite exporter](https://github.com/prometheus/graphite_exporter) 中的说明。 + +`Graphite exporter` 和 `Prometheus exporter` 自身都不带图形界面的，如果需要可视化监控及更高阶的图表分析，请到 `venus-docs` 项目中查找关于 `Prometheus+Grafana` 的说明文档。 From 80b6aeb3b4d09829aca2291b67cee203d1f003d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E9=A1=B5=E7=B4=A0=E4=B9=A6?= <2931107265@qq.com> Date: Wed, 20 Jul 2022 15:42:27 +0800 Subject: [PATCH 8/8] opt: doc & unnecessary code --- docs/zh/metrics-config-desc.md | 7 ++++++- metrics/exporter.go | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/zh/metrics-config-desc.md b/docs/zh/metrics-config-desc.md index 44c64faa..b8c667e2 100644 --- a/docs/zh/metrics-config-desc.md +++ b/docs/zh/metrics-config-desc.md @@ -40,8 +40,9 @@ 如果配置 `Prometheus exporter`，则在 `venus-messager` 服务启动时会附带启动 `Prometheus exporter` 的监听服务，可以通过以下方式快速查看指标： + ```bash - $ curl http://192.168.200.107:4568/debug/metrics + $ curl http://localhost:4568/debug/metrics # HELP messager01_chain_head_stable_dur_s Duration of chain head stabilization # TYPE messager01_chain_head_stable_dur_s histogram messager01_chain_head_stable_dur_s_bucket{le="8"} 0 @@ -63,6 +64,10 @@ messager01_chain_head_stable_s 9 ... ... ``` +> 如果遇到错误 `curl: (56) Recv failure: Connection reset by peer`, 请使用本机 `ip` 地址, 如下所示: +```bash +$ curl http://:4568/debug/metrics +``` 如果配置 `Graphite exporter`，需要先启动 `Graphite exporter` 的收集器服务， `venus-messager` 服务启动时将指标上报给收集器。服务启动参考 [Graphite exporter](https://github.com/prometheus/graphite_exporter) 中的说明。 diff --git a/metrics/exporter.go b/metrics/exporter.go index 05b21321..0fad35b3 100644 --- a/metrics/exporter.go +++ b/metrics/exporter.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "fmt" "go.opencensus.io/stats/view" "go.uber.org/fx" @@ -23,7 +24,7 @@ func SetupMetrics(lc fx.Lifecycle, metricsConfig *metrics.MetricsConfig, log *lo if err := view.Register( MessagerNodeViews..., ); err != nil { - log.Fatalf("Cannot register the view: %v", err) + return fmt.Errorf("cannot register the view: %w", err) } lc.Append(fx.Hook{ @@ -47,9 +48,6 @@ func SetupMetrics(lc fx.Lifecycle, metricsConfig *metrics.MetricsConfig, log *lo return nil }, - OnStop: func(ctx context.Context) error { - return nil - }, }) return nil