metrics: add more metrics and fix some metrics

pkg/manager/cert/manager.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pingcap/TiProxy/lib/util/errors"
 	"github.com/pingcap/TiProxy/lib/util/security"
 	"github.com/pingcap/TiProxy/lib/util/waitgroup"
+	"github.com/pingcap/TiProxy/pkg/metrics"
 	"go.uber.org/zap"
 )
 
@@ -134,8 +135,9 @@ func (cm *CertManager) reload() {
 	if err := cm.sqlTLS.Reload(cm.logger); err != nil {
 		errs = append(errs, err)
 	}
-	err := errors.Collect(errors.New("loading certs"), errs...)
-	if err != nil {
+	if len(errs) > 0 {
+		metrics.ServerErrCounter.WithLabelValues("load_cert").Add(float64(len(errs)))
+		err := errors.Collect(errors.New("loading certs"), errs...)
 		cm.logger.Error("failed to reload some certs", zap.Error(err))
 	}
 }

pkg/manager/router/backend_observer.go

@@ -231,7 +231,9 @@ func (bo *BackendObserver) checkHealth(ctx context.Context, backends map[string]
 
 		// Also dial the SQL port just in case that the SQL port hangs.
 		err := connectWithRetry(func() error {
+			startTime := time.Now()
 			conn, err := net.DialTimeout("tcp", addr, bo.healthCheckConfig.DialTimeout)
+			setPingBackendMetrics(addr, err == nil, startTime)
 			if err == nil {
 				if err := conn.Close(); err != nil && !pnet.IsDisconnectError(err) {
 					bo.logger.Error("close connection in health check failed", zap.Error(err))

pkg/manager/router/metrics.go

@@ -57,9 +57,15 @@ func addMigrateMetrics(from, to string, succeed bool, startTime time.Time) {
 	metrics.MigrateCounter.WithLabelValues(from, to, resLabel).Inc()
 
 	cost := time.Since(startTime)
-	metrics.MigrateDurationHistogram.WithLabelValues(from, to, resLabel).Observe(float64(cost.Milliseconds()))
+	metrics.MigrateDurationHistogram.WithLabelValues(from, to, resLabel).Observe(cost.Seconds())
 }
 
 func readMigrateCounter(from, to string, succeed bool) (int, error) {
 	return metrics.ReadCounter(metrics.MigrateCounter.WithLabelValues(from, to, succeedToLabel(succeed)))
 }
+
+func setPingBackendMetrics(addr string, succeed bool, startTime time.Time) {
+	cost := time.Since(startTime)
+	resLabel := succeedToLabel(succeed)
+	metrics.PingBackendGauge.WithLabelValues(addr, resLabel).Set(cost.Seconds())
+}

pkg/metrics/backend.go

@@ -0,0 +1,57 @@
+// Copyright 2023 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	LblRes    = "res"
+	LblStatus = "status"
+)
+
+var (
+	BackendStatusGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelBackend,
+			Name:      "b_status",
+			Help:      "Gauge of backend status.",
+		}, []string{LblBackend, LblStatus})
+
+	GetBackendHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelBackend,
+			Name:      "get_backend_duration_seconds",
+			Help:      "Bucketed histogram of time (s) for getting an available backend.",
+			Buckets:   prometheus.ExponentialBuckets(0.000001, 2, 26), // 1us ~ 30s
+		})
+
+	GetBackendCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelBackend,
+			Name:      "get_backend",
+			Help:      "Counter of getting backend.",
+		}, []string{LblRes})
+
+	PingBackendGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: ModuleProxy,
+			Subsystem: LabelBackend,
+			Name:      "ping_duration_seconds",
+			Help:      "Time (s) of pinging the SQL port of each backend.",
+		}, []string{LblBackend, LblRes})
+)

pkg/metrics/balance.go

@@ -24,19 +24,10 @@ const (
 	LblBackend       = "backend"
 	LblFrom          = "from"
 	LblTo            = "to"
-	LblStatus        = "status"
 	LblMigrateResult = "migrate_res"
 )
 
 var (
-	BackendStatusGauge = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: ModuleProxy,
-			Subsystem: LabelBalance,
-			Name:      "b_status",
-			Help:      "Gauge of backend status.",
-		}, []string{LblBackend, LblStatus})
-
 	BackendConnGauge = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: ModuleProxy,
@@ -57,8 +48,8 @@ var (
 		prometheus.HistogramOpts{
 			Namespace: ModuleProxy,
 			Subsystem: LabelBalance,
-			Name:      "migrate_duration_millis",
-			Help:      "Bucketed histogram of migrating time (ms) of sessions.",
-			Buckets:   prometheus.ExponentialBuckets(0.1, 2, 26), // 0.1ms ~ 1h
+			Name:      "migrate_duration_seconds",
+			Help:      "Bucketed histogram of migrating time (s) of sessions.",
+			Buckets:   prometheus.ExponentialBuckets(0.0001, 2, 26), // 0.1ms ~ 1h
 		}, []string{LblFrom, LblTo, LblMigrateResult})
 )