Skip to content

Commit

Permalink
*: add metrics to prometheus (#79)
Browse files Browse the repository at this point in the history
  • Loading branch information
djshow832 authored Sep 13, 2022
1 parent b1853c4 commit 349d1d6
Show file tree
Hide file tree
Showing 27 changed files with 701 additions and 303 deletions.
2 changes: 2 additions & 0 deletions lib/config/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type Config struct {
}

type Metrics struct {
MetricsAddr string `toml:"metrics-addr" json:"metrics-addr"`
MetricsInterval uint `toml:"metrics-interval" json:"metrics-interval"`
}

type ProxyServerOnline struct {
Expand Down
5 changes: 4 additions & 1 deletion lib/config/proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ var testProxyConfig = Config{
User: "user",
Password: "pwd",
},
Metrics: Metrics{},
Metrics: Metrics{
MetricsAddr: "127.0.0.1:9021",
MetricsInterval: 15,
},
Log: Log{
Level: "info",
Encoder: "tidb",
Expand Down
4 changes: 3 additions & 1 deletion lib/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/pingcap/log v1.1.0
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.8.0
go.uber.org/atomic v1.9.0
go.uber.org/zap v1.23.0
gopkg.in/yaml.v3 v3.0.1
)
Expand All @@ -15,9 +16,10 @@ require (
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
)
10 changes: 7 additions & 3 deletions lib/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,19 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
Expand Down Expand Up @@ -57,8 +60,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down
40 changes: 40 additions & 0 deletions lib/util/logger/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2022 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
"testing"

"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)

type testingLog struct {
*testing.T
}

func (t *testingLog) Write(b []byte) (int, error) {
t.Logf("%s", b)
return len(b), nil
}

// CreateLoggerForTest creates a logger for unit tests.
func CreateLoggerForTest(t *testing.T) *zap.Logger {
return zap.New(zapcore.NewCore(
zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
zapcore.AddSync(&testingLog{t}),
zap.InfoLevel,
)).Named(t.Name())
}
48 changes: 48 additions & 0 deletions lib/util/systimemon/systime_mon.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package systimemon

import (
"context"
"time"

"go.uber.org/zap"
)

// StartMonitor calls systimeErrHandler if system time jump backward.
func StartMonitor(ctx context.Context, logger *zap.Logger, now func() time.Time, systimeErrHandler func(), successCallback func()) {
logger.Info("start system time monitor")
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()
tickCount := 0
for {
last := now().UnixNano()
select {
case <-tick.C:
case <-ctx.Done():
return
}
if now().UnixNano() < last {
logger.Error("system time jump backward", zap.Int64("last", last))
systimeErrHandler()
}
// call successCallback per second.
tickCount++
if tickCount >= 10 {
tickCount = 0
successCallback()
}
}
}
50 changes: 50 additions & 0 deletions lib/util/systimemon/systime_mon_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package systimemon

import (
"context"
"testing"
"time"

"github.com/pingcap/TiProxy/lib/util/logger"
"github.com/pingcap/TiProxy/lib/util/waitgroup"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
)

func TestSystimeMonitor(t *testing.T) {
errTriggered := atomic.NewBool(false)
nowTriggered := atomic.NewBool(false)
log := logger.CreateLoggerForTest(t)
ctx, cancel := context.WithCancel(context.Background())
var wg waitgroup.WaitGroup
wg.Run(func() {
StartMonitor(ctx, log,
func() time.Time {
if !nowTriggered.Load() {
nowTriggered.Store(true)
return time.Now()
}
return time.Now().Add(-2 * time.Second)
}, func() {
errTriggered.Store(true)
}, func() {})
})

require.Eventually(t, errTriggered.Load, time.Second, 10*time.Millisecond)
cancel()
wg.Wait()
}
21 changes: 4 additions & 17 deletions pkg/manager/config/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,40 +23,27 @@ import (
"testing"

"github.com/pingcap/TiProxy/lib/config"
"github.com/pingcap/TiProxy/lib/util/logger"
"github.com/pingcap/TiProxy/lib/util/waitgroup"
"github.com/stretchr/testify/require"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/server/v3/embed"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)

type testingLog struct {
*testing.T
}

func (t *testingLog) Write(b []byte) (int, error) {
t.Logf("%s", b)
return len(b), nil
}

func testConfigManager(t *testing.T, cfg config.Advance) (*ConfigManager, context.Context) {
addr, err := url.Parse("http://127.0.0.1:0")
require.NoError(t, err)

testDir := t.TempDir()

logger := zap.New(zapcore.NewCore(
zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
zapcore.AddSync(&testingLog{t}),
zap.InfoLevel,
)).Named(t.Name())
log := logger.CreateLoggerForTest(t)

etcd_cfg := embed.NewConfig()
etcd_cfg.LCUrls = []url.URL{*addr}
etcd_cfg.LPUrls = []url.URL{*addr}
etcd_cfg.Dir = filepath.Join(testDir, "etcd")
etcd_cfg.ZapLoggerBuilder = embed.NewZapLoggerBuilder(logger.Named("etcd"))
etcd_cfg.ZapLoggerBuilder = embed.NewZapLoggerBuilder(log.Named("etcd"))
etcd, err := embed.StartEtcd(etcd_cfg)
require.NoError(t, err)

Expand All @@ -71,7 +58,7 @@ func testConfigManager(t *testing.T, cfg config.Advance) (*ConfigManager, contex
}

cfgmgr := NewConfigManager()
require.NoError(t, cfgmgr.Init(ctx, ends, cfg, logger))
require.NoError(t, cfgmgr.Init(ctx, ends, cfg, log))

t.Cleanup(func() {
require.NoError(t, cfgmgr.Close())
Expand Down
11 changes: 9 additions & 2 deletions pkg/manager/router/backend_observer.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const (

var statusNames = map[BackendStatus]string{
StatusHealthy: "healthy",
StatusCannotConnect: "cannot connect",
StatusCannotConnect: "down",
StatusMemoryHigh: "memory high",
StatusRunSlow: "run slow",
StatusSchemaOutdated: "schema outdated",
Expand Down Expand Up @@ -419,15 +419,22 @@ func (bo *BackendObserver) notifyIfChanged(backendStatus map[string]BackendStatu
if lastStatus == StatusHealthy {
if newStatus, ok := backendStatus[addr]; !ok {
updatedBackends[addr] = StatusCannotConnect
updateBackendStatusMetrics(addr, lastStatus, StatusCannotConnect)
} else if newStatus != StatusHealthy {
updatedBackends[addr] = newStatus
updateBackendStatusMetrics(addr, lastStatus, newStatus)
}
}
}
for addr, newStatus := range backendStatus {
if newStatus == StatusHealthy {
if lastStatus, ok := bo.curBackendInfo[addr]; !ok || lastStatus != StatusHealthy {
lastStatus, ok := bo.curBackendInfo[addr]
if !ok {
lastStatus = StatusCannotConnect
}
if lastStatus != StatusHealthy {
updatedBackends[addr] = newStatus
updateBackendStatusMetrics(addr, lastStatus, newStatus)
}
}
}
Expand Down
1 change: 1 addition & 0 deletions pkg/manager/router/backend_observer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ func checkStatus(t *testing.T, backendChan chan map[string]BackendStatus, backen
status, ok := backends[backend.sqlAddr]
require.True(t, ok)
require.Equal(t, expectedStatus, status)
require.True(t, checkBackendStatusMetrics(backend.sqlAddr, status))
}

// Update the TTL for a backend.
Expand Down
65 changes: 65 additions & 0 deletions pkg/manager/router/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright 2022 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
"time"

"github.com/pingcap/TiProxy/pkg/metrics"
)

func updateBackendStatusMetrics(addr string, prevStatus, curStatus BackendStatus) {
metrics.BackendStatusGauge.WithLabelValues(addr, prevStatus.String()).Set(0)
metrics.BackendStatusGauge.WithLabelValues(addr, curStatus.String()).Set(1)
}

func checkBackendStatusMetrics(addr string, status BackendStatus) bool {
val, err := metrics.ReadGauge(metrics.BackendStatusGauge.WithLabelValues(addr, status.String()))
if err != nil {
return false
}
return val == 1
}

func addBackendConnMetrics(addr string) {
metrics.BackendConnGauge.WithLabelValues(addr).Add(1)
}

func subBackendConnMetrics(addr string) {
metrics.BackendConnGauge.WithLabelValues(addr).Sub(1)
}

func readBackendConnMetrics(addr string) (int, error) {
return metrics.ReadGauge(metrics.BackendConnGauge.WithLabelValues(addr))
}

func succeedToLabel(succeed bool) string {
if succeed {
return "succeed"
}
return "fail"
}

func addMigrateMetrics(from, to string, succeed bool, startTime time.Time) {
resLabel := succeedToLabel(succeed)
metrics.MigrateCounter.WithLabelValues(from, to, resLabel).Inc()

cost := time.Since(startTime)
metrics.MigrateDurationHistogram.WithLabelValues(from, to, resLabel).Observe(float64(cost.Milliseconds()))
}

func readMigrateCounter(from, to string, succeed bool) (int, error) {
return metrics.ReadCounter(metrics.MigrateCounter.WithLabelValues(from, to, succeedToLabel(succeed)))
}
Loading

0 comments on commit 349d1d6

Please sign in to comment.