Skip to content

Commit

Permalink
metrics: add uptime and cpu limit to grafana (#257)
Browse files Browse the repository at this point in the history
  • Loading branch information
@djshow832
djshow832 authored Mar 30, 2023
1 parent 05b9446 commit 6c751f9
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 11 deletions.
4 changes: 4 additions & 0 deletions cmd/tiproxy/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ package main
import (
"fmt"
"os"
"runtime"

"github.com/pingcap/TiProxy/lib/util/cmd"
"github.com/pingcap/TiProxy/lib/util/errors"
"github.com/pingcap/TiProxy/pkg/metrics"
"github.com/pingcap/TiProxy/pkg/sctx"
"github.com/pingcap/TiProxy/pkg/server"
"github.com/spf13/cobra"
Expand All @@ -46,6 +48,8 @@ func main() {
rootCmd.PersistentFlags().StringVar(&sctx.Overlay.Log.Encoder, "log_encoder", "", "log in format of tidb, console, or json")
rootCmd.PersistentFlags().StringVar(&sctx.Overlay.Log.Level, "log_level", "", "log level")

metrics.MaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0)))

rootCmd.RunE = func(cmd *cobra.Command, _ []string) error {
srv, err := server.NewServer(cmd.Context(), sctx)
if err != nil {
Expand Down
115 changes: 104 additions & 11 deletions pkg/metrics/grafana/tiproxy_summary.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,13 @@
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"refId": "A"
},
{
"expr": "tiproxy_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "limit-{{instance}}",
"refId": "B"
}
],
"thresholds": [ ],
Expand Down Expand Up @@ -388,6 +395,92 @@
"show": true
}
]
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_TEST-CLUSTER}",
"description": "TiProxy uptime since the last restart.",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 0
},
"id": 7,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sideWidth": null,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [ ],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": null,
"seriesOverrides": [ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", job=\"tiproxy\"}",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeShift": null,
"title": "Uptime",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
Expand All @@ -407,7 +500,7 @@
"x": 0,
"y": 0
},
"id": 7,
"id": 8,
"panels": [
{
"aliasColors": { },
Expand All @@ -424,7 +517,7 @@
"x": 0,
"y": 0
},
"id": 8,
"id": 9,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -524,7 +617,7 @@
"x": 12,
"y": 0
},
"id": 9,
"id": 10,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -610,7 +703,7 @@
"x": 0,
"y": 0
},
"id": 10,
"id": 11,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -696,7 +789,7 @@
"x": 12,
"y": 0
},
"id": 11,
"id": 12,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -785,7 +878,7 @@
"x": 0,
"y": 0
},
"id": 12,
"id": 13,
"panels": [
{
"aliasColors": { },
Expand All @@ -802,7 +895,7 @@
"x": 0,
"y": 0
},
"id": 13,
"id": 14,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -888,7 +981,7 @@
"x": 12,
"y": 0
},
"id": 14,
"id": 15,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -974,7 +1067,7 @@
"x": 0,
"y": 0
},
"id": 15,
"id": 16,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down Expand Up @@ -1077,7 +1170,7 @@
"x": 0,
"y": 0
},
"id": 16,
"id": 17,
"panels": [
{
"aliasColors": { },
Expand All @@ -1094,7 +1187,7 @@
"x": 0,
"y": 0
},
"id": 17,
"id": 18,
"legend": {
"alignAsTable": false,
"avg": false,
Expand Down
21 changes: 21 additions & 0 deletions pkg/metrics/grafana/tiproxy_summary.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@ local cpuP = graphPanel.new(
'rate(process_cpu_seconds_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", job="tiproxy"}[1m])',
legendFormat='{{instance}}',
)
)
.addTarget(
prometheus.target(
'tiproxy_server_maxprocs{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", job="tiproxy"}',
legendFormat='limit-{{instance}}',
)
);

local memP = graphPanel.new(
Expand All @@ -156,6 +162,20 @@ local memP = graphPanel.new(
)
);

local uptimeP = graphPanel.new(
title='Uptime',
datasource=myDS,
legend_rightSide=true,
format='s',
description='TiProxy uptime since the last restart.',
)
.addTarget(
prometheus.target(
'time() - process_start_time_seconds{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", job="tiproxy"}',
legendFormat='{{instance}}',
)
);

// Query Summary
local queryRow = row.new(collapse=true, title='Query Summary');
local durationP = graphPanel.new(
Expand Down Expand Up @@ -327,6 +347,7 @@ newDash
.addPanel(memP, gridPos=rightPanelPos)
.addPanel(connectionP, gridPos=leftPanelPos)
.addPanel(goroutineP, gridPos=rightPanelPos)
.addPanel(uptimeP, gridPos=leftPanelPos)
,
gridPos=rowPos
)
Expand Down
1 change: 1 addition & 0 deletions pkg/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ func registerProxyMetrics() {
prometheus.MustRegister(ConnGauge)
prometheus.MustRegister(TimeJumpBackCounter)
prometheus.MustRegister(KeepAliveCounter)
prometheus.MustRegister(MaxProcsGauge)
prometheus.MustRegister(BackendStatusGauge)
prometheus.MustRegister(BackendConnGauge)
prometheus.MustRegister(QueryTotalCounter)
Expand Down
8 changes: 8 additions & 0 deletions pkg/metrics/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@ var (
Help: "Number of connections.",
})

MaxProcsGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: ModuleProxy,
Subsystem: LabelServer,
Name: "maxprocs",
Help: "The value of GOMAXPROCS.",
})

TimeJumpBackCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: ModuleProxy,
Expand Down

0 comments on commit 6c751f9

Please sign in to comment.