*: add metrics to prometheus

lib/config/proxy.go

@@ -33,6 +33,8 @@ type Config struct {
 }
 
 type Metrics struct {
+	MetricsAddr     string `toml:"metrics-addr" json:"metrics-addr"`
+	MetricsInterval uint   `toml:"metrics-interval" json:"metrics-interval"`
 }
 
 type ProxyServerOnline struct {

lib/config/proxy_test.go

@@ -41,7 +41,10 @@ var testProxyConfig = Config{
 		User:            "user",
 		Password:        "pwd",
 	},
-	Metrics: Metrics{},
+	Metrics: Metrics{
+		MetricsAddr:     "127.0.0.1:9021",
+		MetricsInterval: 15,
+	},
 	Log: Log{
 		Level:   "info",
 		Encoder: "tidb",

lib/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/pingcap/log v1.1.0
 	github.com/spf13/cobra v1.5.0
 	github.com/stretchr/testify v1.8.0
+	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.23.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -15,9 +16,10 @@ require (
 	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 )

lib/go.sum

@@ -3,16 +3,19 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
@@ -57,8 +60,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

lib/util/logger/logger.go

@@ -0,0 +1,40 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logger
+
+import (
+	"testing"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+type testingLog struct {
+	*testing.T
+}
+
+func (t *testingLog) Write(b []byte) (int, error) {
+	t.Logf("%s", b)
+	return len(b), nil
+}
+
+// CreateLoggerForTest creates a logger for unit tests.
+func CreateLoggerForTest(t *testing.T) *zap.Logger {
+	return zap.New(zapcore.NewCore(
+		zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
+		zapcore.AddSync(&testingLog{t}),
+		zap.InfoLevel,
+	)).Named(t.Name())
+}

lib/util/systimemon/systime_mon.go

@@ -0,0 +1,48 @@
+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package systimemon
+
+import (
+	"context"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+// StartMonitor calls systimeErrHandler if system time jump backward.
+func StartMonitor(ctx context.Context, logger *zap.Logger, now func() time.Time, systimeErrHandler func(), successCallback func()) {
+	logger.Info("start system time monitor")
+	tick := time.NewTicker(100 * time.Millisecond)
+	defer tick.Stop()
+	tickCount := 0
+	for {
+		last := now().UnixNano()
+		select {
+		case <-tick.C:
+		case <-ctx.Done():
+			return
+		}
+		if now().UnixNano() < last {
+			logger.Error("system time jump backward", zap.Int64("last", last))
+			systimeErrHandler()
+		}
+		// call successCallback per second.
+		tickCount++
+		if tickCount >= 10 {
+			tickCount = 0
+			successCallback()
+		}
+	}
+}

lib/util/systimemon/systime_mon_test.go

@@ -0,0 +1,50 @@
+// Copyright 2017 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package systimemon
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/pingcap/TiProxy/lib/util/logger"
+	"github.com/pingcap/TiProxy/lib/util/waitgroup"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+)
+
+func TestSystimeMonitor(t *testing.T) {
+	errTriggered := atomic.NewBool(false)
+	nowTriggered := atomic.NewBool(false)
+	log := logger.CreateLoggerForTest(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg waitgroup.WaitGroup
+	wg.Run(func() {
+		StartMonitor(ctx, log,
+			func() time.Time {
+				if !nowTriggered.Load() {
+					nowTriggered.Store(true)
+					return time.Now()
+				}
+				return time.Now().Add(-2 * time.Second)
+			}, func() {
+				errTriggered.Store(true)
+			}, func() {})
+	})
+
+	require.Eventually(t, errTriggered.Load, time.Second, 10*time.Millisecond)
+	cancel()
+	wg.Wait()
+}

pkg/manager/config/manager_test.go

@@ -23,40 +23,27 @@ import (
 	"testing"
 
 	"github.com/pingcap/TiProxy/lib/config"
+	"github.com/pingcap/TiProxy/lib/util/logger"
 	"github.com/pingcap/TiProxy/lib/util/waitgroup"
 	"github.com/stretchr/testify/require"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/server/v3/embed"
 	"go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
 )
 
-type testingLog struct {
-	*testing.T
-}
-
-func (t *testingLog) Write(b []byte) (int, error) {
-	t.Logf("%s", b)
-	return len(b), nil
-}
-
 func testConfigManager(t *testing.T, cfg config.Advance) (*ConfigManager, context.Context) {
 	addr, err := url.Parse("http://127.0.0.1:0")
 	require.NoError(t, err)
 
 	testDir := t.TempDir()
 
-	logger := zap.New(zapcore.NewCore(
-		zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
-		zapcore.AddSync(&testingLog{t}),
-		zap.InfoLevel,
-	)).Named(t.Name())
+	log := logger.CreateLoggerForTest(t)
 
 	etcd_cfg := embed.NewConfig()
 	etcd_cfg.LCUrls = []url.URL{*addr}
 	etcd_cfg.LPUrls = []url.URL{*addr}
 	etcd_cfg.Dir = filepath.Join(testDir, "etcd")
-	etcd_cfg.ZapLoggerBuilder = embed.NewZapLoggerBuilder(logger.Named("etcd"))
+	etcd_cfg.ZapLoggerBuilder = embed.NewZapLoggerBuilder(log.Named("etcd"))
 	etcd, err := embed.StartEtcd(etcd_cfg)
 	require.NoError(t, err)
 
@@ -71,7 +58,7 @@ func testConfigManager(t *testing.T, cfg config.Advance) (*ConfigManager, contex
 	}
 
 	cfgmgr := NewConfigManager()
-	require.NoError(t, cfgmgr.Init(ctx, ends, cfg, logger))
+	require.NoError(t, cfgmgr.Init(ctx, ends, cfg, log))
 
 	t.Cleanup(func() {
 		require.NoError(t, cfgmgr.Close())

pkg/manager/router/backend_observer.go

@@ -61,7 +61,7 @@ const (
 
 var statusNames = map[BackendStatus]string{
 	StatusHealthy:        "healthy",
-	StatusCannotConnect:  "cannot connect",
+	StatusCannotConnect:  "down",
 	StatusMemoryHigh:     "memory high",
 	StatusRunSlow:        "run slow",
 	StatusSchemaOutdated: "schema outdated",
@@ -419,15 +419,22 @@ func (bo *BackendObserver) notifyIfChanged(backendStatus map[string]BackendStatu
 		if lastStatus == StatusHealthy {
 			if newStatus, ok := backendStatus[addr]; !ok {
 				updatedBackends[addr] = StatusCannotConnect
+				updateBackendStatusMetrics(addr, lastStatus, StatusCannotConnect)
 			} else if newStatus != StatusHealthy {
 				updatedBackends[addr] = newStatus
+				updateBackendStatusMetrics(addr, lastStatus, newStatus)
 			}
 		}
 	}
 	for addr, newStatus := range backendStatus {
 		if newStatus == StatusHealthy {
-			if lastStatus, ok := bo.curBackendInfo[addr]; !ok || lastStatus != StatusHealthy {
+			lastStatus, ok := bo.curBackendInfo[addr]
+			if !ok {
+				lastStatus = StatusCannotConnect
+			}
+			if lastStatus != StatusHealthy {
 				updatedBackends[addr] = newStatus
+				updateBackendStatusMetrics(addr, lastStatus, newStatus)
 			}
 		}
 	}

pkg/manager/router/backend_observer_test.go

@@ -364,6 +364,7 @@ func checkStatus(t *testing.T, backendChan chan map[string]BackendStatus, backen
 	status, ok := backends[backend.sqlAddr]
 	require.True(t, ok)
 	require.Equal(t, expectedStatus, status)
+	require.True(t, checkBackendStatusMetrics(backend.sqlAddr, status))
 }
 
 // Update the TTL for a backend.

pkg/manager/router/metrics.go

@@ -0,0 +1,65 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+	"time"
+
+	"github.com/pingcap/TiProxy/pkg/metrics"
+)
+
+func updateBackendStatusMetrics(addr string, prevStatus, curStatus BackendStatus) {
+	metrics.BackendStatusGauge.WithLabelValues(addr, prevStatus.String()).Set(0)
+	metrics.BackendStatusGauge.WithLabelValues(addr, curStatus.String()).Set(1)
+}
+
+func checkBackendStatusMetrics(addr string, status BackendStatus) bool {
+	val, err := metrics.ReadGauge(metrics.BackendStatusGauge.WithLabelValues(addr, status.String()))
+	if err != nil {
+		return false
+	}
+	return val == 1
+}
+
+func addBackendConnMetrics(addr string) {
+	metrics.BackendConnGauge.WithLabelValues(addr).Add(1)
+}
+
+func subBackendConnMetrics(addr string) {
+	metrics.BackendConnGauge.WithLabelValues(addr).Sub(1)
+}
+
+func readBackendConnMetrics(addr string) (int, error) {
+	return metrics.ReadGauge(metrics.BackendConnGauge.WithLabelValues(addr))
+}
+
+func succeedToLabel(succeed bool) string {
+	if succeed {
+		return "succeed"
+	}
+	return "fail"
+}
+
+func addMigrateMetrics(from, to string, succeed bool, startTime time.Time) {
+	resLabel := succeedToLabel(succeed)
+	metrics.MigrateCounter.WithLabelValues(from, to, resLabel).Inc()
+
+	cost := time.Since(startTime)
+	metrics.MigrateDurationHistogram.WithLabelValues(from, to, resLabel).Observe(float64(cost.Milliseconds()))
+}
+
+func readMigrateCounter(from, to string, succeed bool) (int, error) {
+	return metrics.ReadCounter(metrics.MigrateCounter.WithLabelValues(from, to, succeedToLabel(succeed)))
+}