From c2afa14d759671a6737f90551db058227aa0017a Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 13 Jan 2021 12:45:30 -0700 Subject: [PATCH 01/10] *: add infoschema client errors --- errno/infoschema.go | 156 ++++++++++++++++++++++++++++++++++ executor/builder.go | 6 +- executor/infoschema_reader.go | 81 ++++++++++++++++++ executor/simple.go | 3 + go.mod | 2 + go.sum | 3 + infoschema/tables.go | 41 +++++++++ infoschema/tables_test.go | 25 ++++++ server/conn.go | 17 +++- server/driver_tidb.go | 6 ++ server/server_test.go | 61 +++++++++++++ server/tidb_test.go | 4 + 12 files changed, 403 insertions(+), 2 deletions(-) create mode 100644 errno/infoschema.go diff --git a/errno/infoschema.go b/errno/infoschema.go new file mode 100644 index 0000000000000..61a24252090dd --- /dev/null +++ b/errno/infoschema.go @@ -0,0 +1,156 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package errno + +import ( + "sync" + "time" +) + +// The error summary is protected by a mutex for simplicity. +// It is not expected to be hot unless there are concurrent workloads +// that are generating high error/warning counts, in which case +// the system probably has other issues already. + +type errorSummary struct { + sync.Mutex + ErrorCount int + WarningCount int + FirstSeen time.Time + LastSeen time.Time +} + +type globalStats struct { + sync.Mutex + errors map[uint16]*errorSummary +} + +type userStats struct { + sync.Mutex + errors map[string]map[uint16]*errorSummary +} + +type hostStats struct { + sync.Mutex + errors map[string]map[uint16]*errorSummary +} + +var global globalStats +var users userStats +var hosts hostStats + +func init() { + global.errors = make(map[uint16]*errorSummary) + users.errors = make(map[string]map[uint16]*errorSummary) + hosts.errors = make(map[string]map[uint16]*errorSummary) +} + +// FlushStats resets errors and warnings across global/users/hosts +func FlushStats() { + global.Lock() + defer global.Unlock() + users.Lock() + defer users.Unlock() + hosts.Lock() + defer hosts.Unlock() + + global.errors = make(map[uint16]*errorSummary) + users.errors = make(map[string]map[uint16]*errorSummary) + hosts.errors = make(map[string]map[uint16]*errorSummary) +} + +// GlobalStats summarizes errors and warnings across all users/hosts +func GlobalStats() map[uint16]*errorSummary { + global.Lock() + defer global.Unlock() + return global.errors +} + +// UserStats summarizes per-user +func UserStats() map[string]map[uint16]*errorSummary { + users.Lock() + defer users.Unlock() + return users.errors +} + +// HostStats summarizes per remote-host +func HostStats() map[string]map[uint16]*errorSummary { + hosts.Lock() + defer hosts.Unlock() + return hosts.errors +} + +func initCounters(errCode uint16, user, host string) { + global.Lock() + if _, ok := global.errors[errCode]; !ok { + global.errors[errCode] = &errorSummary{FirstSeen: time.Now()} + } + global.Unlock() + users.Lock() + if _, ok := users.errors[user]; !ok { + users.errors[user] = make(map[uint16]*errorSummary) + } + if _, ok := users.errors[user][errCode]; !ok { + users.errors[user][errCode] = &errorSummary{FirstSeen: time.Now()} + } + users.Unlock() + hosts.Lock() + if _, ok := hosts.errors[host]; !ok { + hosts.errors[host] = make(map[uint16]*errorSummary) + } + if _, ok := hosts.errors[host][errCode]; !ok { + hosts.errors[host][errCode] = &errorSummary{FirstSeen: time.Now()} + } + hosts.Unlock() +} + +// IncrementError increments the global/user/host statistics for an errCode +func IncrementError(errCode uint16, user, host string) { + initCounters(errCode, user, host) + // Increment counter + update last seen + global.errors[errCode].Lock() + global.errors[errCode].ErrorCount++ + global.errors[errCode].LastSeen = time.Now() + global.errors[errCode].Unlock() + // Increment counter + update last seen + users.errors[user][errCode].Lock() + users.errors[user][errCode].ErrorCount++ + users.errors[user][errCode].LastSeen = time.Now() + users.errors[user][errCode].Unlock() + // Increment counter + update last seen + hosts.errors[host][errCode].Lock() + hosts.errors[host][errCode].ErrorCount++ + hosts.errors[host][errCode].LastSeen = time.Now() + hosts.errors[host][errCode].Unlock() +} + +// IncrementWarning increments the global/user/host statistics for an errCode +func IncrementWarning(errCode uint16, user, host string) { + initCounters(errCode, user, host) + // Increment counter + update last seen + global.errors[errCode].Lock() + global.errors[errCode].WarningCount++ + global.errors[errCode].LastSeen = time.Now() + global.errors[errCode].Unlock() + // Increment counter + update last seen + users.errors[user][errCode].Lock() + users.errors[user][errCode].WarningCount++ + users.errors[user][errCode].LastSeen = time.Now() + users.errors[user][errCode].Unlock() + // Increment counter + update last seen + hosts.errors[host][errCode].Lock() + hosts.errors[host][errCode].WarningCount++ + hosts.errors[host][errCode].LastSeen = time.Now() + hosts.errors[host][errCode].Unlock() +} diff --git a/executor/builder.go b/executor/builder.go index 7fce1d4c71a37..ed888ec3e8454 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1533,7 +1533,11 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo strings.ToLower(infoschema.TableStatementsSummaryHistory), strings.ToLower(infoschema.ClusterTableStatementsSummary), strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory), - strings.ToLower(infoschema.TablePlacementPolicy): + strings.ToLower(infoschema.TablePlacementPolicy), + strings.ToLower(infoschema.TablePlacementPolicy), + strings.ToLower(infoschema.TableClientErrorsSummaryGlobal), + strings.ToLower(infoschema.TableClientErrorsSummaryByUser), + strings.ToLower(infoschema.TableClientErrorsSummaryByHost): return &MemTableReaderExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ID()), table: v.Table, diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index cf733da680c0f..55ff005f80478 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/domain/infosync" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/meta/autoid" plannercore "github.com/pingcap/tidb/planner/core" @@ -142,6 +143,10 @@ func (e *memtableRetriever) retrieve(ctx context.Context, sctx sessionctx.Contex err = e.setDataForStatementsSummary(sctx, e.table.Name.O) case infoschema.TablePlacementPolicy: err = e.setDataForPlacementPolicy(sctx) + case infoschema.TableClientErrorsSummaryGlobal, + infoschema.TableClientErrorsSummaryByUser, + infoschema.TableClientErrorsSummaryByHost: + err = e.setDataForClientErrorsSummary(sctx, e.table.Name.O) } if err != nil { return nil, err @@ -1864,6 +1869,82 @@ func (e *memtableRetriever) setDataForPlacementPolicy(ctx sessionctx.Context) er return nil } +func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context, tableName string) error { + // Seeing client errors should require the PROCESS privilege, with the exception of errors for your own user. + // This is similar to information_schema.processlist, which is the closest comparison. + var hasProcessPriv bool + loginUser := ctx.GetSessionVars().User + if pm := privilege.GetPrivilegeManager(ctx); pm != nil { + if pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, "", "", "", mysql.ProcessPriv) { + hasProcessPriv = true + } + } + + var rows [][]types.Datum + switch tableName { + case infoschema.TableClientErrorsSummaryGlobal: + if !hasProcessPriv { + return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") + } + for code, summary := range errno.GlobalStats() { + firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) + lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) + row := types.MakeDatums( + int(code), // ERROR_NUMBER + errno.MySQLErrName[code].Raw, // ERROR_MESSAGE + summary.ErrorCount, // ERROR_COUNT + summary.WarningCount, // WARNING_COUNT + firstSeen, // FIRST_SEEN + lastSeen, // LAST_SEEN + ) + rows = append(rows, row) + } + case infoschema.TableClientErrorsSummaryByUser: + for user, agg := range errno.UserStats() { + for code, summary := range agg { + // Allow anyone to see their own errors. + if !hasProcessPriv && loginUser != nil && loginUser.Username != user { + continue + } + firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) + lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) + row := types.MakeDatums( + user, // USER + int(code), // ERROR_NUMBER + errno.MySQLErrName[code].Raw, // ERROR_MESSAGE + summary.ErrorCount, // ERROR_COUNT + summary.WarningCount, // WARNING_COUNT + firstSeen, // FIRST_SEEN + lastSeen, // LAST_SEEN + ) + rows = append(rows, row) + } + } + case infoschema.TableClientErrorsSummaryByHost: + if !hasProcessPriv { + return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") + } + for host, agg := range errno.HostStats() { + for code, summary := range agg { + firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) + lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) + row := types.MakeDatums( + host, // HOST + int(code), // ERROR_NUMBER + errno.MySQLErrName[code].Raw, // ERROR_MESSAGE + summary.ErrorCount, // ERROR_COUNT + summary.WarningCount, // WARNING_COUNT + firstSeen, // FIRST_SEEN + lastSeen, // LAST_SEEN + ) + rows = append(rows, row) + } + } + } + e.rows = rows + return nil +} + type hugeMemTableRetriever struct { dummyCloser table *model.TableInfo diff --git a/executor/simple.go b/executor/simple.go index 8e46aee85aac6..6b7c174f869ab 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" @@ -1261,6 +1262,8 @@ func (e *SimpleExec) executeFlush(s *ast.FlushStmt) error { return err } } + case ast.FlushClientErrorsSummary: + errno.FlushStats() } return nil } diff --git a/go.mod b/go.mod index 63a7da37d93ff..a64580e0148eb 100644 --- a/go.mod +++ b/go.mod @@ -85,3 +85,5 @@ require ( ) go 1.13 + +replace github.com/pingcap/parser => github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999 diff --git a/go.sum b/go.sum index 533849047442b..8cc798e0edd23 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,8 @@ github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJ github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999 h1:8IfXx2ql9BTOV1yLoxV5hNrIeDj0wEdPGP+IrjLYMA8= +github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -739,6 +741,7 @@ github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrY github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.20.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= diff --git a/infoschema/tables.go b/infoschema/tables.go index 477ee0aef567d..2b0de938d5e5d 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -155,6 +155,12 @@ const ( TableTiFlashSegments = "TIFLASH_SEGMENTS" // TablePlacementPolicy is the string constant of placement policy table. TablePlacementPolicy = "PLACEMENT_POLICY" + // TableClientErrorsSummaryGlobal is the string constant of client errors table. + TableClientErrorsSummaryGlobal = "CLIENT_ERRORS_SUMMARY_GLOBAL" + // TableClientErrorsSummaryByUser is the string constant of client errors table. + TableClientErrorsSummaryByUser = "CLIENT_ERRORS_SUMMARY_BY_USER" + // TableClientErrorsSummaryByHost is the string constant of client errors table. + TableClientErrorsSummaryByHost = "CLIENT_ERRORS_SUMMARY_BY_HOST" ) var tableIDMap = map[string]int64{ @@ -224,6 +230,9 @@ var tableIDMap = map[string]int64{ TableTiFlashTables: autoid.InformationSchemaDBID + 64, TableTiFlashSegments: autoid.InformationSchemaDBID + 65, TablePlacementPolicy: autoid.InformationSchemaDBID + 66, + TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, + TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, + TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, } type columnInfo struct { @@ -1289,6 +1298,35 @@ var tablePlacementPolicyCols = []columnInfo{ {name: "CONSTRAINTS", tp: mysql.TypeVarchar, size: 1024}, } +var tableClientErrorsSummaryGlobalCols = []columnInfo{ + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, + {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, +} + +var tableClientErrorsSummaryByUserCols = []columnInfo{ + {name: "USER", tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, + {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, +} + +var tableClientErrorsSummaryByHostCols = []columnInfo{ + {name: "HOST", tp: mysql.TypeVarchar, size: 255, flag: mysql.NotNullFlag}, + {name: "ERROR_NUMBER", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "ERROR_MESSAGE", tp: mysql.TypeVarchar, size: 1024, flag: mysql.NotNullFlag}, + {name: "ERROR_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "WARNING_COUNT", tp: mysql.TypeLonglong, size: 64, flag: mysql.NotNullFlag}, + {name: "FIRST_SEEN", tp: mysql.TypeTimestamp, size: 26}, + {name: "LAST_SEEN", tp: mysql.TypeTimestamp, size: 26}, +} + // GetShardingInfo returns a nil or description string for the sharding information of given TableInfo. // The returned description string may be: // - "NOT_SHARDED": for tables that SHARD_ROW_ID_BITS is not specified. @@ -1655,6 +1693,9 @@ var tableNameToColumns = map[string][]columnInfo{ TableTiFlashTables: tableTableTiFlashTablesCols, TableTiFlashSegments: tableTableTiFlashSegmentsCols, TablePlacementPolicy: tablePlacementPolicyCols, + TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, + TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, + TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, } func createInfoSchemaTable(_ autoid.Allocators, meta *model.TableInfo) (table.Table, error) { diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 32f347a1f9d79..a2608b31cd25e 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/placement" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" @@ -1460,3 +1461,27 @@ func (s *testTableSuite) TestPlacementPolicy(c *C) { tk.MustQuery("select rule_id, schema_name, table_name, partition_name from information_schema.placement_policy order by partition_name, rule_id").Check(testkit.Rows( "0 test test_placement p0", "1 test test_placement p0", "0 test test_placement p1", "1 test test_placement p1")) } + +func (s *testTableSuite) TestInfoschemaClientErrors(c *C) { + tk := s.newTestKitWithRoot(c) + + tk.MustExec("FLUSH CLIENT_ERRORS_SUMMARY") + + errno.IncrementError(1365, "root", "localhost") + errno.IncrementError(1365, "infoschematest", "localhost") + errno.IncrementError(1365, "root", "localhost") + + tk.MustExec("CREATE USER 'infoschematest'@'localhost'") + c.Assert(tk.Se.Auth(&auth.UserIdentity{Username: "infoschematest", Hostname: "localhost"}, nil, nil), IsTrue) + + err := tk.QueryToErr("SELECT * FROM information_schema.client_errors_summary_global") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation") + + err = tk.QueryToErr("SELECT * FROM information_schema.client_errors_summary_by_host") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the PROCESS privilege(s) for this operation") + + tk.MustQuery("SELECT error_number, error_count, warning_count FROM information_schema.client_errors_summary_by_user ORDER BY error_number").Check(testkit.Rows("1365 1 0")) + + err = tk.ExecToErr("FLUSH CLIENT_ERRORS_SUMMARY") + c.Assert(err.Error(), Equals, "[planner:1227]Access denied; you need (at least one of) the RELOAD privilege(s) for this operation") +} diff --git a/server/conn.go b/server/conn.go index 7df21dd5f759f..2d131c256cdd5 100644 --- a/server/conn.go +++ b/server/conn.go @@ -52,6 +52,8 @@ import ( "time" "unsafe" + goerr "errors" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -62,6 +64,7 @@ import ( "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/errno" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -1081,7 +1084,18 @@ func (cc *clientConn) useDB(ctx context.Context, db string) (err error) { } func (cc *clientConn) flush(ctx context.Context) error { - defer trace.StartRegion(ctx, "FlushClientConn").End() + defer func() { + trace.StartRegion(ctx, "FlushClientConn").End() + if cc.ctx != nil && cc.ctx.WarningCount() > 0 { + for _, err := range cc.ctx.GetWarnings() { + var warn *errors.Error + if ok := goerr.As(err.Err, &warn); ok { + code := uint16(warn.Code()) + errno.IncrementWarning(code, cc.user, cc.peerHost) + } + } + } + }() failpoint.Inject("FakeClientConn", func() { if cc.pkt == nil { failpoint.Return(nil) @@ -1143,6 +1157,7 @@ func (cc *clientConn) writeError(ctx context.Context, e error) error { } cc.lastCode = m.Code + defer errno.IncrementError(m.Code, cc.user, cc.peerHost) data := cc.alloc.AllocWithLen(4, 16+len(m.Message)) data = append(data, mysql.ErrHeader) data = append(data, byte(m.Code), byte(m.Code>>8)) diff --git a/server/driver_tidb.go b/server/driver_tidb.go index 8af42fca5ba96..ae8fe74b87878 100644 --- a/server/driver_tidb.go +++ b/server/driver_tidb.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/sqlexec" @@ -197,6 +198,11 @@ func (qd *TiDBDriver) OpenCtx(connID uint64, capability uint32, collation uint8, return tc, nil } +// GetWarnings implements QueryCtx GetWarnings method. +func (tc *TiDBContext) GetWarnings() []stmtctx.SQLWarn { + return tc.GetSessionVars().StmtCtx.GetWarnings() +} + // CurrentDB implements QueryCtx CurrentDB method. func (tc *TiDBContext) CurrentDB() string { return tc.currentDB diff --git a/server/server_test.go b/server/server_test.go index 4d402c108cbf9..67c51cb04e6e3 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1737,3 +1737,64 @@ func (cli *testServerClient) waitUntilServerOnline() { log.Fatal("failed to connect HTTP status in every 10 ms", zap.Int("retryTime", retryTime)) } } + +// Client errors are only incremented when using the TiDB Server protocol, +// and not internal SQL statements. Thus, this test is in the server-test suite. +func (cli *testServerClient) runTestInfoschemaClientErrors(t *C) { + cli.runTestsOnNewDB(t, nil, "clientErrors", func(dbt *DBTest) { + + clientErrors := []struct { + stmt string + incrementWarnings bool + incrementErrors bool + errCode int + }{ + { + stmt: "SELECT 0/0", + incrementWarnings: true, + errCode: 1365, // div by zero + }, + { + stmt: "CREATE TABLE test_client_errors2 (a int primary key, b int primary key)", + incrementErrors: true, + errCode: 1068, // multiple pkeys + }, + { + stmt: "gibberish", + incrementErrors: true, + errCode: 1064, // parse error + }, + } + + sources := []string{"client_errors_summary_global", "client_errors_summary_by_user", "client_errors_summary_by_host"} + + for _, test := range clientErrors { + for _, tbl := range sources { + + var errors, warnings int + rows := dbt.mustQuery("SELECT SUM(error_count), SUM(warning_count) FROM information_schema."+tbl+" WHERE error_number = ? GROUP BY error_number", test.errCode) + if rows.Next() { + rows.Scan(&errors, &warnings) + } + + if test.incrementErrors { + errors++ + } + if test.incrementWarnings { + warnings++ + } + + dbt.db.Query(test.stmt) // ignore results and errors (query table) + var newErrors, newWarnings int + rows = dbt.mustQuery("SELECT SUM(error_count), SUM(warning_count) FROM information_schema."+tbl+" WHERE error_number = ? GROUP BY error_number", test.errCode) + if rows.Next() { + rows.Scan(&newErrors, &newWarnings) + } + + dbt.Check(newErrors, Equals, errors) + dbt.Check(newWarnings, Equals, warnings) + } + } + + }) +} diff --git a/server/tidb_test.go b/server/tidb_test.go index 1a6dbe27b488f..10177e59bf53e 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -938,6 +938,10 @@ func (ts *tidbTestSuite) TestFieldList(c *C) { c.Assert(cols[0].Name, Equals, columnAsName) } +func (ts *tidbTestSuite) TestClientErrors(c *C) { + ts.runTestInfoschemaClientErrors(c) +} + func (ts *tidbTestSuite) TestSumAvg(c *C) { c.Parallel() ts.runTestSumAvg(c) From 1ec76983259cdbebf0b19c133ad86406af16fb94 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 20 Jan 2021 22:40:31 -0700 Subject: [PATCH 02/10] Update for parser PR merging --- go.mod | 14 ++++++-------- go.sum | 23 ++++++++++++++++------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index a64580e0148eb..19dde241396d2 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 github.com/pingcap/kvproto v0.0.0-20201215060142-f3dafca4c7fd github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 - github.com/pingcap/parser v0.0.0-20210108074737-814a888e05e2 + github.com/pingcap/parser v0.0.0-20210121043122-1c9989974a2b github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible github.com/pingcap/tipb v0.0.0-20201229060814-148bc717ce4c @@ -70,20 +70,18 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect - golang.org/x/net v0.0.0-20200904194848-62affa334b73 + golang.org/x/net v0.0.0-20201021035429-f5854403a974 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20200819171115-d785dc25833f - golang.org/x/text v0.3.4 - golang.org/x/tools v0.0.0-20200820010801-b793a1359eac + golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 + golang.org/x/text v0.3.5 + golang.org/x/tools v0.1.0 google.golang.org/grpc v1.27.1 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.3.0 // indirect - honnef.co/go/tools v0.1.0 // indirect + honnef.co/go/tools v0.1.1 // indirect sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 ) go 1.13 - -replace github.com/pingcap/parser => github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999 diff --git a/go.sum b/go.sum index 8cc798e0edd23..56d459369ebce 100644 --- a/go.sum +++ b/go.sum @@ -544,8 +544,6 @@ github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJ github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999 h1:8IfXx2ql9BTOV1yLoxV5hNrIeDj0wEdPGP+IrjLYMA8= -github.com/morgo/parser v0.0.0-20210113194243-aa72bd5cf999/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -655,8 +653,8 @@ github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 h1:M+DNpOu/I3uDmwee6vc github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/parser v0.0.0-20200422082501-7329d80eaf2c/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= -github.com/pingcap/parser v0.0.0-20210108074737-814a888e05e2 h1:8ESOr7xSfwM3jdWlklug0Mz8tjpb/KzDh+YSJx2SI8w= -github.com/pingcap/parser v0.0.0-20210108074737-814a888e05e2/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= +github.com/pingcap/parser v0.0.0-20210121043122-1c9989974a2b h1:34RafWnLQIee2kFxZ7TkT3ZuU2HU6+Zd5FQ2nZgb0WI= +github.com/pingcap/parser v0.0.0-20210121043122-1c9989974a2b/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE= github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= @@ -741,7 +739,6 @@ github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrY github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.20.3+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -892,6 +889,7 @@ github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= github.com/zaf/temp v0.0.0-20170209143821-94e385923345/go.mod h1:sXsZgXwh6DB0qlskmZVB4HE93e5YrktMrgUDPy9iYmY= github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018 h1:T3OrqVdcH6z6SakR7WkECvGpdkfB0MAur/6zf66GPxQ= @@ -1024,6 +1022,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1083,12 +1083,15 @@ golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f h1:KJuwZVtZBVzDmEDtB2zro9CXkD9O0dpCv4o2LHbQIAw= golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1142,10 +1145,14 @@ golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200609164405-eb789aa7ce50/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200820010801-b793a1359eac h1:DugppSxw0LSF8lcjaODPJZoDzq0ElTGskTst3ZaBkHI= golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -1259,6 +1266,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.0 h1:AWNL1W1i7f0wNZ8VwOKNJ0sliKvOF/adn0EHenfUh+c= honnef.co/go/tools v0.1.0/go.mod h1:XtegFAyX/PfluP4921rXU5IkjkqBCDnUq4W8VCIoKvM= +honnef.co/go/tools v0.1.1 h1:EVDuO03OCZwpV2t/tLLxPmPiomagMoBOgfPt0FM+4IY= +honnef.co/go/tools v0.1.1/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= From aa10cb72c554b508e8a970b582efe3bc97d9daa3 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 20 Jan 2021 22:47:43 -0700 Subject: [PATCH 03/10] Run go mod tidy --- go.sum | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/go.sum b/go.sum index 56d459369ebce..66ca450be4f09 100644 --- a/go.sum +++ b/go.sum @@ -888,7 +888,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20181031023651-12c4817b42c5/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= github.com/zaf/temp v0.0.0-20170209143821-94e385923345/go.mod h1:sXsZgXwh6DB0qlskmZVB4HE93e5YrktMrgUDPy9iYmY= @@ -1019,9 +1018,6 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1036,7 +1032,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1081,8 +1076,6 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200819171115-d785dc25833f h1:KJuwZVtZBVzDmEDtB2zro9CXkD9O0dpCv4o2LHbQIAw= -golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1142,9 +1135,6 @@ golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200422205258-72e4a01eba43/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200609164405-eb789aa7ce50/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200820010801-b793a1359eac h1:DugppSxw0LSF8lcjaODPJZoDzq0ElTGskTst3ZaBkHI= -golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1264,8 +1254,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.0 h1:AWNL1W1i7f0wNZ8VwOKNJ0sliKvOF/adn0EHenfUh+c= -honnef.co/go/tools v0.1.0/go.mod h1:XtegFAyX/PfluP4921rXU5IkjkqBCDnUq4W8VCIoKvM= honnef.co/go/tools v0.1.1 h1:EVDuO03OCZwpV2t/tLLxPmPiomagMoBOgfPt0FM+4IY= honnef.co/go/tools v0.1.1/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= From 3608c9ee6de31c4d57181a60e0e3b2afd639e27f Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 26 Jan 2021 10:21:16 -0700 Subject: [PATCH 04/10] Address reviewer feedback --- errno/infoschema.go | 134 +++++++++++++++++----------------- executor/builder.go | 1 - executor/infoschema_reader.go | 15 +++- 3 files changed, 81 insertions(+), 69 deletions(-) diff --git a/errno/infoschema.go b/errno/infoschema.go index 61a24252090dd..b896178a93a31 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -23,7 +23,8 @@ import ( // that are generating high error/warning counts, in which case // the system probably has other issues already. -type errorSummary struct { +// ErrorSummary summarizes errors and warnings +type ErrorSummary struct { sync.Mutex ErrorCount int WarningCount int @@ -31,29 +32,32 @@ type errorSummary struct { LastSeen time.Time } -type globalStats struct { +// GlobalStats summarizes errors and warnings globally +type GlobalStats struct { sync.Mutex - errors map[uint16]*errorSummary + Errors map[uint16]*ErrorSummary } -type userStats struct { +// UserStats summarizes errors and warnings per user +type UserStats struct { sync.Mutex - errors map[string]map[uint16]*errorSummary + Errors map[string]map[uint16]*ErrorSummary } -type hostStats struct { +// HostStats summarizes errors and warnings per host +type HostStats struct { sync.Mutex - errors map[string]map[uint16]*errorSummary + Errors map[string]map[uint16]*ErrorSummary } -var global globalStats -var users userStats -var hosts hostStats +var global GlobalStats +var users UserStats +var hosts HostStats func init() { - global.errors = make(map[uint16]*errorSummary) - users.errors = make(map[string]map[uint16]*errorSummary) - hosts.errors = make(map[string]map[uint16]*errorSummary) + global.Errors = make(map[uint16]*ErrorSummary) + users.Errors = make(map[string]map[uint16]*ErrorSummary) + hosts.Errors = make(map[string]map[uint16]*ErrorSummary) } // FlushStats resets errors and warnings across global/users/hosts @@ -65,92 +69,92 @@ func FlushStats() { hosts.Lock() defer hosts.Unlock() - global.errors = make(map[uint16]*errorSummary) - users.errors = make(map[string]map[uint16]*errorSummary) - hosts.errors = make(map[string]map[uint16]*errorSummary) + global.Errors = make(map[uint16]*ErrorSummary) + users.Errors = make(map[string]map[uint16]*ErrorSummary) + hosts.Errors = make(map[string]map[uint16]*ErrorSummary) } -// GlobalStats summarizes errors and warnings across all users/hosts -func GlobalStats() map[uint16]*errorSummary { - global.Lock() - defer global.Unlock() - return global.errors +// GetGlobalStats summarizes errors and warnings across all users/hosts +// It should be read while held under a lock. +func GetGlobalStats() *GlobalStats { + return &global } -// UserStats summarizes per-user -func UserStats() map[string]map[uint16]*errorSummary { - users.Lock() - defer users.Unlock() - return users.errors +// GetUserStats summarizes per-user +// It should be read while held under a lock. +func GetUserStats() *UserStats { + return &users } -// HostStats summarizes per remote-host -func HostStats() map[string]map[uint16]*errorSummary { - hosts.Lock() - defer hosts.Unlock() - return hosts.errors +// GetHostStats summarizes per remote-host +// It should be read while held under a lock. +func GetHostStats() *HostStats { + return &hosts } func initCounters(errCode uint16, user, host string) { + seen := time.Now() global.Lock() - if _, ok := global.errors[errCode]; !ok { - global.errors[errCode] = &errorSummary{FirstSeen: time.Now()} + if _, ok := global.Errors[errCode]; !ok { + global.Errors[errCode] = &ErrorSummary{FirstSeen: seen} } global.Unlock() users.Lock() - if _, ok := users.errors[user]; !ok { - users.errors[user] = make(map[uint16]*errorSummary) + if _, ok := users.Errors[user]; !ok { + users.Errors[user] = make(map[uint16]*ErrorSummary) } - if _, ok := users.errors[user][errCode]; !ok { - users.errors[user][errCode] = &errorSummary{FirstSeen: time.Now()} + if _, ok := users.Errors[user][errCode]; !ok { + users.Errors[user][errCode] = &ErrorSummary{FirstSeen: seen} } users.Unlock() hosts.Lock() - if _, ok := hosts.errors[host]; !ok { - hosts.errors[host] = make(map[uint16]*errorSummary) + if _, ok := hosts.Errors[host]; !ok { + hosts.Errors[host] = make(map[uint16]*ErrorSummary) } - if _, ok := hosts.errors[host][errCode]; !ok { - hosts.errors[host][errCode] = &errorSummary{FirstSeen: time.Now()} + if _, ok := hosts.Errors[host][errCode]; !ok { + hosts.Errors[host][errCode] = &ErrorSummary{FirstSeen: seen} } hosts.Unlock() } // IncrementError increments the global/user/host statistics for an errCode func IncrementError(errCode uint16, user, host string) { + seen := time.Now() initCounters(errCode, user, host) // Increment counter + update last seen - global.errors[errCode].Lock() - global.errors[errCode].ErrorCount++ - global.errors[errCode].LastSeen = time.Now() - global.errors[errCode].Unlock() + global.Errors[errCode].Lock() + global.Errors[errCode].ErrorCount++ + global.Errors[errCode].LastSeen = seen + global.Errors[errCode].Unlock() // Increment counter + update last seen - users.errors[user][errCode].Lock() - users.errors[user][errCode].ErrorCount++ - users.errors[user][errCode].LastSeen = time.Now() - users.errors[user][errCode].Unlock() + users.Errors[user][errCode].Lock() + users.Errors[user][errCode].ErrorCount++ + users.Errors[user][errCode].LastSeen = seen + users.Errors[user][errCode].Unlock() // Increment counter + update last seen - hosts.errors[host][errCode].Lock() - hosts.errors[host][errCode].ErrorCount++ - hosts.errors[host][errCode].LastSeen = time.Now() - hosts.errors[host][errCode].Unlock() + hosts.Errors[host][errCode].Lock() + hosts.Errors[host][errCode].ErrorCount++ + hosts.Errors[host][errCode].LastSeen = seen + hosts.Errors[host][errCode].Unlock() } // IncrementWarning increments the global/user/host statistics for an errCode func IncrementWarning(errCode uint16, user, host string) { + seen := time.Now() initCounters(errCode, user, host) // Increment counter + update last seen - global.errors[errCode].Lock() - global.errors[errCode].WarningCount++ - global.errors[errCode].LastSeen = time.Now() - global.errors[errCode].Unlock() + global.Errors[errCode].Lock() + global.Errors[errCode].WarningCount++ + global.Errors[errCode].LastSeen = seen + global.Errors[errCode].Unlock() // Increment counter + update last seen - users.errors[user][errCode].Lock() - users.errors[user][errCode].WarningCount++ - users.errors[user][errCode].LastSeen = time.Now() - users.errors[user][errCode].Unlock() + users.Errors[user][errCode].Lock() + users.Errors[user][errCode].WarningCount++ + users.Errors[user][errCode].LastSeen = seen + users.Errors[user][errCode].Unlock() // Increment counter + update last seen - hosts.errors[host][errCode].Lock() - hosts.errors[host][errCode].WarningCount++ - hosts.errors[host][errCode].LastSeen = time.Now() - hosts.errors[host][errCode].Unlock() + hosts.Errors[host][errCode].Lock() + hosts.Errors[host][errCode].WarningCount++ + hosts.Errors[host][errCode].LastSeen = seen + hosts.Errors[host][errCode].Unlock() } diff --git a/executor/builder.go b/executor/builder.go index 8452e333ebc92..cfd2d5f271726 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1534,7 +1534,6 @@ func (b *executorBuilder) buildMemTable(v *plannercore.PhysicalMemTable) Executo strings.ToLower(infoschema.ClusterTableStatementsSummary), strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory), strings.ToLower(infoschema.TablePlacementPolicy), - strings.ToLower(infoschema.TablePlacementPolicy), strings.ToLower(infoschema.TableClientErrorsSummaryGlobal), strings.ToLower(infoschema.TableClientErrorsSummaryByUser), strings.ToLower(infoschema.TableClientErrorsSummaryByHost): diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index 87ad204815358..c02d24750147e 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -1891,7 +1891,9 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context if !hasProcessPriv { return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } - for code, summary := range errno.GlobalStats() { + global := errno.GetGlobalStats() + global.Lock() + for code, summary := range global.Errors { firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) row := types.MakeDatums( @@ -1904,8 +1906,11 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context ) rows = append(rows, row) } + global.Unlock() case infoschema.TableClientErrorsSummaryByUser: - for user, agg := range errno.UserStats() { + users := errno.GetUserStats() + users.Lock() + for user, agg := range users.Errors { for code, summary := range agg { // Allow anyone to see their own errors. if !hasProcessPriv && loginUser != nil && loginUser.Username != user { @@ -1925,11 +1930,14 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context rows = append(rows, row) } } + users.Unlock() case infoschema.TableClientErrorsSummaryByHost: if !hasProcessPriv { return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } - for host, agg := range errno.HostStats() { + hosts := errno.GetHostStats() + hosts.Lock() + for host, agg := range hosts.Errors { for code, summary := range agg { firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) @@ -1945,6 +1953,7 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context rows = append(rows, row) } } + hosts.Unlock() } e.rows = rows return nil From ba137ef7481ac32473c5eacd637339539ba0c91d Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 27 Jan 2021 12:01:24 -0700 Subject: [PATCH 05/10] change to single mutex + use deep copy --- errno/infoschema.go | 163 +++++++++++++++++----------------- executor/infoschema_reader.go | 15 +--- 2 files changed, 83 insertions(+), 95 deletions(-) diff --git a/errno/infoschema.go b/errno/infoschema.go index b896178a93a31..a79f7476df3a8 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -32,89 +32,86 @@ type ErrorSummary struct { LastSeen time.Time } -// GlobalStats summarizes errors and warnings globally -type GlobalStats struct { +// instanceStatistics provide statistics for a tidb-server instance. +type instanceStatistics struct { sync.Mutex - Errors map[uint16]*ErrorSummary + global map[uint16]*ErrorSummary + users map[string]map[uint16]*ErrorSummary + hosts map[string]map[uint16]*ErrorSummary } -// UserStats summarizes errors and warnings per user -type UserStats struct { - sync.Mutex - Errors map[string]map[uint16]*ErrorSummary -} - -// HostStats summarizes errors and warnings per host -type HostStats struct { - sync.Mutex - Errors map[string]map[uint16]*ErrorSummary -} - -var global GlobalStats -var users UserStats -var hosts HostStats +var stats instanceStatistics func init() { - global.Errors = make(map[uint16]*ErrorSummary) - users.Errors = make(map[string]map[uint16]*ErrorSummary) - hosts.Errors = make(map[string]map[uint16]*ErrorSummary) + FlushStats() } // FlushStats resets errors and warnings across global/users/hosts func FlushStats() { - global.Lock() - defer global.Unlock() - users.Lock() - defer users.Unlock() - hosts.Lock() - defer hosts.Unlock() - - global.Errors = make(map[uint16]*ErrorSummary) - users.Errors = make(map[string]map[uint16]*ErrorSummary) - hosts.Errors = make(map[string]map[uint16]*ErrorSummary) + stats.Lock() + defer stats.Unlock() + stats.global = make(map[uint16]*ErrorSummary) + stats.users = make(map[string]map[uint16]*ErrorSummary) + stats.hosts = make(map[string]map[uint16]*ErrorSummary) } -// GetGlobalStats summarizes errors and warnings across all users/hosts -// It should be read while held under a lock. -func GetGlobalStats() *GlobalStats { - return &global +func copyMap(oldMap map[uint16]*ErrorSummary) map[uint16]*ErrorSummary { + newMap := make(map[uint16]*ErrorSummary, len(oldMap)) + for k, v := range oldMap { + newMap[k] = v + } + return newMap +} + +// GlobalStats summarizes errors and warnings across all users/hosts +func GlobalStats() map[uint16]*ErrorSummary { + stats.Lock() + defer stats.Unlock() + return copyMap(stats.global) } -// GetUserStats summarizes per-user -// It should be read while held under a lock. -func GetUserStats() *UserStats { - return &users +// UserStats summarizes per-user +func UserStats() map[string]map[uint16]*ErrorSummary { + stats.Lock() + defer stats.Unlock() + newMap := make(map[string]map[uint16]*ErrorSummary) + for k, v := range stats.users { + newMap[k] = copyMap(v) + } + return newMap } -// GetHostStats summarizes per remote-host -// It should be read while held under a lock. -func GetHostStats() *HostStats { - return &hosts +// HostStats summarizes per remote-host +func HostStats() map[string]map[uint16]*ErrorSummary { + stats.Lock() + defer stats.Unlock() + newMap := make(map[string]map[uint16]*ErrorSummary) + for k, v := range stats.hosts { + newMap[k] = copyMap(v) + } + return newMap } func initCounters(errCode uint16, user, host string) { seen := time.Now() - global.Lock() - if _, ok := global.Errors[errCode]; !ok { - global.Errors[errCode] = &ErrorSummary{FirstSeen: seen} + stats.Lock() + defer stats.Unlock() + + if _, ok := stats.global[errCode]; !ok { + stats.global[errCode] = &ErrorSummary{FirstSeen: seen} } - global.Unlock() - users.Lock() - if _, ok := users.Errors[user]; !ok { - users.Errors[user] = make(map[uint16]*ErrorSummary) + if _, ok := stats.users[user]; !ok { + stats.users[user] = make(map[uint16]*ErrorSummary) } - if _, ok := users.Errors[user][errCode]; !ok { - users.Errors[user][errCode] = &ErrorSummary{FirstSeen: seen} + if _, ok := stats.users[user][errCode]; !ok { + stats.users[user][errCode] = &ErrorSummary{FirstSeen: seen} } - users.Unlock() - hosts.Lock() - if _, ok := hosts.Errors[host]; !ok { - hosts.Errors[host] = make(map[uint16]*ErrorSummary) + if _, ok := stats.hosts[host]; !ok { + stats.hosts[host] = make(map[uint16]*ErrorSummary) } - if _, ok := hosts.Errors[host][errCode]; !ok { - hosts.Errors[host][errCode] = &ErrorSummary{FirstSeen: seen} + if _, ok := stats.hosts[host][errCode]; !ok { + stats.hosts[host][errCode] = &ErrorSummary{FirstSeen: seen} } - hosts.Unlock() } // IncrementError increments the global/user/host statistics for an errCode @@ -122,20 +119,20 @@ func IncrementError(errCode uint16, user, host string) { seen := time.Now() initCounters(errCode, user, host) // Increment counter + update last seen - global.Errors[errCode].Lock() - global.Errors[errCode].ErrorCount++ - global.Errors[errCode].LastSeen = seen - global.Errors[errCode].Unlock() + stats.global[errCode].Lock() + stats.global[errCode].ErrorCount++ + stats.global[errCode].LastSeen = seen + stats.global[errCode].Unlock() // Increment counter + update last seen - users.Errors[user][errCode].Lock() - users.Errors[user][errCode].ErrorCount++ - users.Errors[user][errCode].LastSeen = seen - users.Errors[user][errCode].Unlock() + stats.users[user][errCode].Lock() + stats.users[user][errCode].ErrorCount++ + stats.users[user][errCode].LastSeen = seen + stats.users[user][errCode].Unlock() // Increment counter + update last seen - hosts.Errors[host][errCode].Lock() - hosts.Errors[host][errCode].ErrorCount++ - hosts.Errors[host][errCode].LastSeen = seen - hosts.Errors[host][errCode].Unlock() + stats.hosts[host][errCode].Lock() + stats.hosts[host][errCode].ErrorCount++ + stats.hosts[host][errCode].LastSeen = seen + stats.hosts[host][errCode].Unlock() } // IncrementWarning increments the global/user/host statistics for an errCode @@ -143,18 +140,18 @@ func IncrementWarning(errCode uint16, user, host string) { seen := time.Now() initCounters(errCode, user, host) // Increment counter + update last seen - global.Errors[errCode].Lock() - global.Errors[errCode].WarningCount++ - global.Errors[errCode].LastSeen = seen - global.Errors[errCode].Unlock() + stats.global[errCode].Lock() + stats.global[errCode].WarningCount++ + stats.global[errCode].LastSeen = seen + stats.global[errCode].Unlock() // Increment counter + update last seen - users.Errors[user][errCode].Lock() - users.Errors[user][errCode].WarningCount++ - users.Errors[user][errCode].LastSeen = seen - users.Errors[user][errCode].Unlock() + stats.users[user][errCode].Lock() + stats.users[user][errCode].WarningCount++ + stats.users[user][errCode].LastSeen = seen + stats.users[user][errCode].Unlock() // Increment counter + update last seen - hosts.Errors[host][errCode].Lock() - hosts.Errors[host][errCode].WarningCount++ - hosts.Errors[host][errCode].LastSeen = seen - hosts.Errors[host][errCode].Unlock() + stats.hosts[host][errCode].Lock() + stats.hosts[host][errCode].WarningCount++ + stats.hosts[host][errCode].LastSeen = seen + stats.hosts[host][errCode].Unlock() } diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index c02d24750147e..87ad204815358 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -1891,9 +1891,7 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context if !hasProcessPriv { return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } - global := errno.GetGlobalStats() - global.Lock() - for code, summary := range global.Errors { + for code, summary := range errno.GlobalStats() { firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) row := types.MakeDatums( @@ -1906,11 +1904,8 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context ) rows = append(rows, row) } - global.Unlock() case infoschema.TableClientErrorsSummaryByUser: - users := errno.GetUserStats() - users.Lock() - for user, agg := range users.Errors { + for user, agg := range errno.UserStats() { for code, summary := range agg { // Allow anyone to see their own errors. if !hasProcessPriv && loginUser != nil && loginUser.Username != user { @@ -1930,14 +1925,11 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context rows = append(rows, row) } } - users.Unlock() case infoschema.TableClientErrorsSummaryByHost: if !hasProcessPriv { return plannercore.ErrSpecificAccessDenied.GenWithStackByArgs("PROCESS") } - hosts := errno.GetHostStats() - hosts.Lock() - for host, agg := range hosts.Errors { + for host, agg := range errno.HostStats() { for code, summary := range agg { firstSeen := types.NewTime(types.FromGoTime(summary.FirstSeen), mysql.TypeTimestamp, types.DefaultFsp) lastSeen := types.NewTime(types.FromGoTime(summary.LastSeen), mysql.TypeTimestamp, types.DefaultFsp) @@ -1953,7 +1945,6 @@ func (e *memtableRetriever) setDataForClientErrorsSummary(ctx sessionctx.Context rows = append(rows, row) } } - hosts.Unlock() } e.rows = rows return nil From 3c960e921214c871ecedc6e0d38dbe96d13ec519 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 27 Jan 2021 12:13:40 -0700 Subject: [PATCH 06/10] change to single mutex --- errno/infoschema.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/errno/infoschema.go b/errno/infoschema.go index a79f7476df3a8..6ae1c24664f84 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -25,7 +25,6 @@ import ( // ErrorSummary summarizes errors and warnings type ErrorSummary struct { - sync.Mutex ErrorCount int WarningCount int FirstSeen time.Time @@ -118,40 +117,36 @@ func initCounters(errCode uint16, user, host string) { func IncrementError(errCode uint16, user, host string) { seen := time.Now() initCounters(errCode, user, host) + + stats.Lock() + defer stats.Unlock() + // Increment counter + update last seen - stats.global[errCode].Lock() stats.global[errCode].ErrorCount++ stats.global[errCode].LastSeen = seen - stats.global[errCode].Unlock() // Increment counter + update last seen - stats.users[user][errCode].Lock() stats.users[user][errCode].ErrorCount++ stats.users[user][errCode].LastSeen = seen - stats.users[user][errCode].Unlock() // Increment counter + update last seen - stats.hosts[host][errCode].Lock() stats.hosts[host][errCode].ErrorCount++ stats.hosts[host][errCode].LastSeen = seen - stats.hosts[host][errCode].Unlock() } // IncrementWarning increments the global/user/host statistics for an errCode func IncrementWarning(errCode uint16, user, host string) { seen := time.Now() initCounters(errCode, user, host) + + stats.Lock() + defer stats.Unlock() + // Increment counter + update last seen - stats.global[errCode].Lock() stats.global[errCode].WarningCount++ stats.global[errCode].LastSeen = seen - stats.global[errCode].Unlock() // Increment counter + update last seen - stats.users[user][errCode].Lock() stats.users[user][errCode].WarningCount++ stats.users[user][errCode].LastSeen = seen - stats.users[user][errCode].Unlock() // Increment counter + update last seen - stats.hosts[host][errCode].Lock() stats.hosts[host][errCode].WarningCount++ stats.hosts[host][errCode].LastSeen = seen - stats.hosts[host][errCode].Unlock() } From aa6d46c8d5289de072b8ad05ebaac39386e5abfa Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 28 Jan 2021 16:18:51 -0700 Subject: [PATCH 07/10] Fix deep copy, add tests --- errno/infoschema.go | 7 +++- errno/infoschema_test.go | 72 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 errno/infoschema_test.go diff --git a/errno/infoschema.go b/errno/infoschema.go index 6ae1c24664f84..5ea81775676b9 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -57,7 +57,12 @@ func FlushStats() { func copyMap(oldMap map[uint16]*ErrorSummary) map[uint16]*ErrorSummary { newMap := make(map[uint16]*ErrorSummary, len(oldMap)) for k, v := range oldMap { - newMap[k] = v + newMap[k] = &ErrorSummary{ + ErrorCount: v.ErrorCount, + WarningCount: v.WarningCount, + FirstSeen: v.FirstSeen, + LastSeen: v.LastSeen, + } } return newMap } diff --git a/errno/infoschema_test.go b/errno/infoschema_test.go new file mode 100644 index 0000000000000..02befced27909 --- /dev/null +++ b/errno/infoschema_test.go @@ -0,0 +1,72 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package errno + +import ( + "testing" + + . "github.com/pingcap/check" +) + +func TestT(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&testErrno{}) + +type testErrno struct{} + +func (s *testErrno) TestGetSetSysVars(c *C) { + + IncrementError(123, "user", "host") + IncrementError(321, "user2", "host2") + + globalCopy := GlobalStats() + userCopy := UserStats() + hostCopy := HostStats() + + IncrementError(123, "user", "host") + IncrementError(999, "user2", "host2") + IncrementError(123, "user3", "host") + + // global stats + c.Assert(stats.global[123].ErrorCount, Equals, 3) + c.Assert(globalCopy[123].ErrorCount, Equals, 1) + + // user stats + c.Assert(len(stats.users), Equals, 3) + c.Assert(len(userCopy), Equals, 2) + c.Assert(stats.users["user"][123].ErrorCount, Equals, 2) + c.Assert(userCopy["user"][123].ErrorCount, Equals, 1) + + // ensure there is no user3 in userCopy + _, ok := userCopy["user3"] + c.Assert(ok, IsFalse) + _, ok = stats.users["user3"] + c.Assert(ok, IsTrue) + + // host stats + c.Assert(len(stats.hosts), Equals, 2) + c.Assert(len(hostCopy), Equals, 2) + IncrementError(123, "user3", "newhost") + c.Assert(len(stats.hosts), Equals, 3) + c.Assert(len(hostCopy), Equals, 2) + + // ensure there is no newhost in hostCopy + _, ok = hostCopy["newhost"] + c.Assert(ok, IsFalse) + _, ok = stats.hosts["newhost"] + c.Assert(ok, IsTrue) + +} From 0a1f635935b3e1712a81ef04877aa2ba2f37ef96 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 28 Jan 2021 16:28:00 -0700 Subject: [PATCH 08/10] Improve test coverage to 100% --- errno/infoschema_test.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/errno/infoschema_test.go b/errno/infoschema_test.go index 02befced27909..1a8d0f0c3984c 100644 --- a/errno/infoschema_test.go +++ b/errno/infoschema_test.go @@ -27,10 +27,13 @@ var _ = Suite(&testErrno{}) type testErrno struct{} -func (s *testErrno) TestGetSetSysVars(c *C) { +func (s *testErrno) TestCopySafety(c *C) { IncrementError(123, "user", "host") IncrementError(321, "user2", "host2") + IncrementWarning(123, "user", "host") + IncrementWarning(999, "user", "host") + IncrementWarning(222, "u", "h") globalCopy := GlobalStats() userCopy := UserStats() @@ -39,34 +42,48 @@ func (s *testErrno) TestGetSetSysVars(c *C) { IncrementError(123, "user", "host") IncrementError(999, "user2", "host2") IncrementError(123, "user3", "host") + IncrementWarning(123, "user", "host") + IncrementWarning(222, "u", "h") + IncrementWarning(222, "a", "b") + IncrementWarning(333, "c", "d") // global stats c.Assert(stats.global[123].ErrorCount, Equals, 3) c.Assert(globalCopy[123].ErrorCount, Equals, 1) // user stats - c.Assert(len(stats.users), Equals, 3) - c.Assert(len(userCopy), Equals, 2) + c.Assert(len(stats.users), Equals, 6) + c.Assert(len(userCopy), Equals, 3) c.Assert(stats.users["user"][123].ErrorCount, Equals, 2) + c.Assert(stats.users["user"][123].WarningCount, Equals, 2) c.Assert(userCopy["user"][123].ErrorCount, Equals, 1) + c.Assert(userCopy["user"][123].WarningCount, Equals, 1) // ensure there is no user3 in userCopy _, ok := userCopy["user3"] c.Assert(ok, IsFalse) _, ok = stats.users["user3"] c.Assert(ok, IsTrue) + _, ok = userCopy["a"] + c.Assert(ok, IsFalse) + _, ok = stats.users["a"] + c.Assert(ok, IsTrue) // host stats - c.Assert(len(stats.hosts), Equals, 2) - c.Assert(len(hostCopy), Equals, 2) + c.Assert(len(stats.hosts), Equals, 5) + c.Assert(len(hostCopy), Equals, 3) IncrementError(123, "user3", "newhost") - c.Assert(len(stats.hosts), Equals, 3) - c.Assert(len(hostCopy), Equals, 2) + c.Assert(len(stats.hosts), Equals, 6) + c.Assert(len(hostCopy), Equals, 3) // ensure there is no newhost in hostCopy _, ok = hostCopy["newhost"] c.Assert(ok, IsFalse) _, ok = stats.hosts["newhost"] c.Assert(ok, IsTrue) + _, ok = hostCopy["b"] + c.Assert(ok, IsFalse) + _, ok = stats.hosts["b"] + c.Assert(ok, IsTrue) } From c62a2ea62b32ff8fefc902467aa59b27fd4ff89b Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Sat, 30 Jan 2021 23:26:28 -0700 Subject: [PATCH 09/10] Update errno/infoschema.go Co-authored-by: djshow832 --- errno/infoschema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/errno/infoschema.go b/errno/infoschema.go index 5ea81775676b9..a0d7c9513abf3 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -78,7 +78,7 @@ func GlobalStats() map[uint16]*ErrorSummary { func UserStats() map[string]map[uint16]*ErrorSummary { stats.Lock() defer stats.Unlock() - newMap := make(map[string]map[uint16]*ErrorSummary) + newMap := make(map[string]map[uint16]*ErrorSummary, len(stats.users)) for k, v := range stats.users { newMap[k] = copyMap(v) } From acf1f513f1cf7c63eab0d42cef86337b5ef49f1d Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Sat, 30 Jan 2021 23:26:33 -0700 Subject: [PATCH 10/10] Update errno/infoschema.go Co-authored-by: djshow832 --- errno/infoschema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/errno/infoschema.go b/errno/infoschema.go index a0d7c9513abf3..60eb5358c83b6 100644 --- a/errno/infoschema.go +++ b/errno/infoschema.go @@ -89,7 +89,7 @@ func UserStats() map[string]map[uint16]*ErrorSummary { func HostStats() map[string]map[uint16]*ErrorSummary { stats.Lock() defer stats.Unlock() - newMap := make(map[string]map[uint16]*ErrorSummary) + newMap := make(map[string]map[uint16]*ErrorSummary, len(stats.hosts)) for k, v := range stats.hosts { newMap[k] = copyMap(v) }