diff --git a/DEPS.bzl b/DEPS.bzl index 1a28f81807e64..999f92e039f4c 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -7037,13 +7037,13 @@ def go_deps(): name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "fc2042d3b3c753de90ac2afdfea97b663f3043aa716f1ee56d2fdf98864e3cbd", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240914083230-71f6f96816e9", + sha256 = "ddfcb88a8b79c238b08c15c88f8142211cb9f64435c5ad371f682d8c81c8cc42", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240926021936-642f0e919b0d", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240914083230-71f6f96816e9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240926021936-642f0e919b0d.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index 6829dca916796..d48cbc66710e2 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tidwall/btree v1.7.0 github.com/tikv/client-go/v2 v2.0.8-0.20240925070302-58f3322fc39a - github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 + github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible @@ -187,7 +187,6 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/apache/thrift v0.16.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.14.3 github.com/bmatcuk/doublestar/v2 v2.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 7f632fafee4c1..469ec613b2017 100644 --- a/go.sum +++ b/go.sum @@ -826,8 +826,8 @@ github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tikv/client-go/v2 v2.0.8-0.20240925070302-58f3322fc39a h1:uKwqdjp/XgoziDJmu200NXwm94X7J5cDSA66cULHy9o= github.com/tikv/client-go/v2 v2.0.8-0.20240925070302-58f3322fc39a/go.mod h1:JZq2+O34RjrronQ9+sSrnPuUT0BHFMjayg11Sosi1JY= -github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 h1:J9LChGMzo95eBrjE03NHITDWgxfPgskH+QrCnlW61/Y= -github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= +github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d h1:iBIi3+grJWLNI5c7BLVezgOonYtL/EQqSUDsPVK2kUw= +github.com/tikv/pd/client v0.0.0-20240926021936-642f0e919b0d/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= diff --git a/pkg/domain/domain_sysvars.go b/pkg/domain/domain_sysvars.go index 8314cb7e4516f..22132ee08784f 100644 --- a/pkg/domain/domain_sysvars.go +++ b/pkg/domain/domain_sysvars.go @@ -83,6 +83,24 @@ func (do *Domain) setPDClientDynamicOption(name, sVal string) error { return err } variable.EnablePDFollowerHandleRegion.Store(val) + case variable.TiDBTSOClientRPCMode: + var concurrency int + + switch sVal { + case variable.TSOClientRPCModeDefault: + concurrency = 1 + case variable.TSOClientRPCModeParallel: + concurrency = 2 + case variable.TSOClientRPCModeParallelFast: + concurrency = 4 + default: + return variable.ErrWrongValueForVar.GenWithStackByArgs(name, sVal) + } + + err := do.updatePDClient(pd.TSOClientRPCConcurrency, concurrency) + if err != nil { + return err + } } return nil } diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index c460f1d5360f8..3ab3f07a652aa 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -12343,6 +12343,13 @@ "intervalFactor": 2, "legendFormat": "90", "refId": "C" + }, + { + "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[1m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"wait\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D" } ], "thresholds": [], @@ -12458,6 +12465,13 @@ "intervalFactor": 2, "legendFormat": "99", "refId": "C" + }, + { + "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[1m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"tso\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D" } ], "thresholds": [], @@ -12502,6 +12516,107 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The estimated latency of TSO RPC calls that's used to adjust batching time for parallel RPC requests", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 23763572011, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "pd_client_request_estimate_tso_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{stream}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Estimate TSO RTT Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 9e407b7eb1870..fc0c1d81de872 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -3399,6 +3399,11 @@ var defaultSysVars = []*SysVar{ s.SharedLockPromotion = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal, Name: TiDBTSOClientRPCMode, Value: DefTiDBTSOClientRPCMode, Type: TypeEnum, PossibleValues: []string{TSOClientRPCModeDefault, TSOClientRPCModeParallel, TSOClientRPCModeParallelFast}, + SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + return (*SetPDClientDynamicOption.Load())(TiDBTSOClientRPCMode, val) + }, + }, } // GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store) diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 9a89085aa8b2b..fb05c65e8f930 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1212,6 +1212,9 @@ const ( // TiDBEnableLazyCursorFetch defines whether to enable the lazy cursor fetch. If it's `OFF`, all results of // of a cursor will be stored in the tidb node in `EXECUTE` command. TiDBEnableLazyCursorFetch = "tidb_enable_lazy_cursor_fetch" + // TiDBTSOClientRPCMode controls how the TSO client performs the TSO RPC requests. It internally controls the + // concurrency of the RPC. This variable provides an approach to tune the latency of getting timestamps from PD. + TiDBTSOClientRPCMode = "tidb_tso_client_rpc_mode" ) // TiDB intentional limits @@ -1560,6 +1563,7 @@ const ( DefTiDBEnableLazyCursorFetch = false DefOptEnableProjectionPushDown = true DefTiDBEnableSharedLockPromotion = false + DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault ) // Process global variables. diff --git a/pkg/sessionctx/variable/variable.go b/pkg/sessionctx/variable/variable.go index 5eb94f379a0f4..aceb862f8f77b 100644 --- a/pkg/sessionctx/variable/variable.go +++ b/pkg/sessionctx/variable/variable.go @@ -83,6 +83,18 @@ const ( OOMActionCancel = "CANCEL" // OOMActionLog constants represents the valid action configurations for OOMAction "LOG". OOMActionLog = "LOG" + + // TSOClientRPCModeDefault is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client sends batched + // TSO requests serially. + TSOClientRPCModeDefault = "DEFAULT" + // TSOClientRPCModeParallel is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client tries to + // keep approximately 2 batched TSO requests running in parallel. This option tries to reduce the batch-waiting time + // by half, at the expense of about twice the amount of TSO RPC calls. + TSOClientRPCModeParallel = "PARALLEL" + // TSOClientRPCModeParallelFast is a choice of variable TiDBTSOClientRPCMode. In this mode, the TSO client tries to + // keep approximately 4 batched TSO requests running in parallel. This option tries to reduce the batch-waiting time + // by 3/4, at the expense of about 4 times the amount of TSO RPC calls. + TSOClientRPCModeParallelFast = "PARALLEL-FAST" ) // Global config name list.