Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@qw4990
qw4990 authored Dec 23, 2019
2 parents f30685d + f9cdebd commit fb547ba
Show file tree
Hide file tree
Showing 259 changed files with 113,951 additions and 2,294 deletions.
26 changes: 21 additions & 5 deletions bindinfo/bind_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,9 +496,9 @@ func (s *testSuite) TestUseMultiplyBindings(c *C) {
tk.MustExec("create binding for select * from t where a >= 1 and b >= 1 and c = 0 using select * from t use index(idx_a) where a >= 1 and b >= 1 and c = 0")
tk.MustExec("create binding for select * from t where a >= 1 and b >= 1 and c = 0 using select * from t use index(idx_b) where a >= 1 and b >= 1 and c = 0")
// It cannot choose table path although it has lowest cost.
tk.MustQuery("select * from t where a >= 4 and b >= 1 and c = 0")
tk.MustQuery("select * from t where a >= 4 and b >= 1 and c = 0;")
c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_a")
tk.MustQuery("select * from t where a >= 1 and b >= 4 and c = 0")
tk.MustQuery("select * from t where a >= 1 and b >= 4 and c = 0;")
c.Assert(tk.Se.GetSessionVars().StmtCtx.IndexNames[0], Equals, "t:idx_b")
}

Expand All @@ -520,10 +520,12 @@ func (s *testSuite) TestDropSingleBindings(c *C) {
rows = tk.MustQuery("show bindings").Rows()
c.Assert(len(rows), Equals, 1)
c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)")
tk.MustExec("drop table t")
tk.MustExec("drop binding for select * from t using select * from t use index(idx_b)")
rows = tk.MustQuery("show bindings").Rows()
c.Assert(len(rows), Equals, 0)

tk.MustExec("create table t(a int, b int, c int, index idx_a(a), index idx_b(b))")
// Test drop global bindings.
tk.MustExec("create global binding for select * from t using select * from t use index(idx_a)")
tk.MustExec("create global binding for select * from t using select * from t use index(idx_b)")
Expand All @@ -535,6 +537,7 @@ func (s *testSuite) TestDropSingleBindings(c *C) {
rows = tk.MustQuery("show global bindings").Rows()
c.Assert(len(rows), Equals, 1)
c.Assert(rows[0][1], Equals, "select * from t use index(idx_b)")
tk.MustExec("drop table t")
tk.MustExec("drop global binding for select * from t using select * from t use index(idx_b)")
rows = tk.MustQuery("show global bindings").Rows()
c.Assert(len(rows), Equals, 0)
Expand Down Expand Up @@ -572,18 +575,31 @@ func (s *testSuite) TestBindingCache(c *C) {
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b int, index idx(a))")
tk.MustExec("create global binding for select * from t using select * from t use index(idx)")
tk.MustExec("create global binding for select * from t using select * from t use index(idx);")
tk.MustExec("create database tmp")
tk.MustExec("use tmp")
tk.MustExec("create table t(a int, b int, index idx(a))")
tk.MustExec("create global binding for select * from t using select * from t use index(idx)")
tk.MustExec("create global binding for select * from t using select * from t use index(idx);")

c.Assert(s.domain.BindHandle().Update(false), IsNil)
c.Assert(s.domain.BindHandle().Update(false), IsNil)
res := tk.MustQuery("show global bindings")
c.Assert(len(res.Rows()), Equals, 2)

tk.MustExec("drop global binding for select * from t")
tk.MustExec("drop global binding for select * from t;")
c.Assert(s.domain.BindHandle().Update(false), IsNil)
c.Assert(len(s.domain.BindHandle().GetAllBindRecord()), Equals, 1)
}

func (s *testSuite) TestDefaultSessionVars(c *C) {
tk := testkit.NewTestKit(c, s.store)
s.cleanBindingEnv(tk)
tk.MustQuery(`show variables like "%baselines%"`).Sort().Check(testkit.Rows(
"tidb_capture_plan_baselines off",
"tidb_evolve_plan_baselines off",
"tidb_use_plan_baselines on"))
tk.MustQuery(`show global variables like "%baselines%"`).Sort().Check(testkit.Rows(
"tidb_capture_plan_baselines off",
"tidb_evolve_plan_baselines off",
"tidb_use_plan_baselines on"))
}
20 changes: 14 additions & 6 deletions bindinfo/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ type Binding struct {
id string
}

func (b *Binding) isSame(rb *Binding) bool {
if b.id != "" && rb.id != "" {
return b.id == rb.id
}
// Sometimes we cannot construct `id` because of the changed schema, so we need to compare by bind sql.
return b.BindSQL == rb.BindSQL
}

// cache is a k-v map, key is original sql, value is a slice of BindRecord.
type cache map[string][]*BindRecord

Expand Down Expand Up @@ -90,7 +98,7 @@ func (br *BindRecord) FindBinding(hint string) *Binding {
func (br *BindRecord) prepareHints(sctx sessionctx.Context, is infoschema.InfoSchema) error {
p := parser.New()
for i, bind := range br.Bindings {
if bind.Hint != nil || bind.id != "" {
if bind.Hint != nil || bind.id != "" || bind.Status == deleted {
continue
}
stmtNode, err := p.ParseOneStmt(bind.BindSQL, bind.Charset, bind.Collation)
Expand Down Expand Up @@ -119,7 +127,7 @@ func merge(lBindRecord, rBindRecord *BindRecord) *BindRecord {
for _, rbind := range rBindRecord.Bindings {
found := false
for j, lbind := range lBindRecord.Bindings {
if lbind.id == rbind.id {
if lbind.isSame(&rbind) {
found = true
if rbind.UpdateTime.Compare(lbind.UpdateTime) >= 0 {
result.Bindings[j] = rbind
Expand All @@ -142,7 +150,7 @@ func (br *BindRecord) remove(deleted *BindRecord) *BindRecord {
result := br.shallowCopy()
for _, deletedBind := range deleted.Bindings {
for i, bind := range result.Bindings {
if bind.id == deletedBind.id {
if bind.isSame(&deletedBind) {
result.Bindings = append(result.Bindings[:i], result.Bindings[i+1:]...)
break
}
Expand Down Expand Up @@ -199,14 +207,14 @@ func (br *BindRecord) metrics() ([]float64, []int) {
sizes[statusIndex[br.Bindings[0].Status]] = commonLength
for _, binding := range br.Bindings {
sizes[statusIndex[binding.Status]] += binding.size()
count[statusIndex[binding.Status]] += 1
count[statusIndex[binding.Status]]++
}
return sizes, count
}

// size calculates the memory size of a bind info.
func (m *Binding) size() float64 {
res := len(m.BindSQL) + len(m.Status) + 2*int(unsafe.Sizeof(m.CreateTime)) + len(m.Charset) + len(m.Collation)
func (b *Binding) size() float64 {
res := len(b.BindSQL) + len(b.Status) + 2*int(unsafe.Sizeof(b.CreateTime)) + len(b.Charset) + len(b.Collation)
return float64(res)
}

Expand Down
52 changes: 19 additions & 33 deletions bindinfo/handle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ package bindinfo

import (
"context"
"errors"
"fmt"
"runtime"
"strconv"
Expand Down Expand Up @@ -105,9 +104,7 @@ func NewBindHandle(ctx sessionctx.Context) *BindHandle {
handle.bindInfo.parser = parser.New()
handle.invalidBindRecordMap.Value.Store(make(map[string]*bindRecordUpdate))
handle.invalidBindRecordMap.flushFunc = func(record *BindRecord) error {
// We do not need the first two parameters because they are only use to generate hint,
// and we already have the hint.
return handle.DropBindRecord(nil, nil, record)
return handle.DropBindRecord(record.OriginalSQL, record.Db, &record.Bindings[0])
}
handle.pendingVerifyBindRecordMap.Value.Store(make(map[string]*bindRecordUpdate))
handle.pendingVerifyBindRecordMap.flushFunc = func(record *BindRecord) error {
Expand Down Expand Up @@ -249,13 +246,9 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, is infoschema.InfoSc
}

// DropBindRecord drops a BindRecord to the storage and BindRecord int the cache.
func (h *BindHandle) DropBindRecord(sctx sessionctx.Context, is infoschema.InfoSchema, record *BindRecord) (err error) {
err = record.prepareHints(sctx, is)
if err != nil {
return err
}
exec, _ := h.sctx.Context.(sqlexec.SQLExecutor)
func (h *BindHandle) DropBindRecord(originalSQL, db string, binding *Binding) (err error) {
h.sctx.Lock()
exec, _ := h.sctx.Context.(sqlexec.SQLExecutor)

_, err = exec.Execute(context.TODO(), "BEGIN")
if err != nil {
Expand All @@ -277,7 +270,11 @@ func (h *BindHandle) DropBindRecord(sctx sessionctx.Context, is infoschema.InfoS
return
}

err = h.removeBindRecord(parser.DigestNormalized(record.OriginalSQL), record)
record := &BindRecord{OriginalSQL: originalSQL, Db: db}
if binding != nil {
record.Bindings = append(record.Bindings, *binding)
}
err = h.removeBindRecord(parser.DigestNormalized(originalSQL), record)
}()

txn, err1 := h.sctx.Context.Txn(true)
Expand All @@ -290,21 +287,13 @@ func (h *BindHandle) DropBindRecord(sctx sessionctx.Context, is infoschema.InfoS
Type: mysql.TypeDatetime,
Fsp: 3,
}
oldBindRecord := h.GetBindRecord(parser.DigestNormalized(record.OriginalSQL), record.OriginalSQL, record.Db)
bindingSQLs := make([]string, 0, len(record.Bindings))
for i := range record.Bindings {
record.Bindings[i].Status = deleted
record.Bindings[i].UpdateTime = updateTs
if oldBindRecord == nil {
continue
}
binding := oldBindRecord.FindBinding(record.Bindings[i].id)
if binding != nil {
bindingSQLs = append(bindingSQLs, binding.BindSQL)
}

bindSQL := ""
if binding != nil {
bindSQL = binding.BindSQL
}

_, err = exec.Execute(context.TODO(), h.logicalDeleteBindInfoSQL(record.OriginalSQL, record.Db, updateTs, bindingSQLs))
_, err = exec.Execute(context.TODO(), h.logicalDeleteBindInfoSQL(originalSQL, db, updateTs, bindSQL))
return err
}

Expand Down Expand Up @@ -523,19 +512,16 @@ func (h *BindHandle) insertBindInfoSQL(orignalSQL string, db string, info Bindin
)
}

func (h *BindHandle) logicalDeleteBindInfoSQL(originalSQL, db string, updateTs types.Time, bindingSQLs []string) string {
func (h *BindHandle) logicalDeleteBindInfoSQL(originalSQL, db string, updateTs types.Time, bindingSQL string) string {
sql := fmt.Sprintf(`UPDATE mysql.bind_info SET status=%s,update_time=%s WHERE original_sql=%s and default_db=%s`,
expression.Quote(deleted),
expression.Quote(updateTs.String()),
expression.Quote(originalSQL),
expression.Quote(db))
if len(bindingSQLs) == 0 {
if bindingSQL == "" {
return sql
}
for i, sql := range bindingSQLs {
bindingSQLs[i] = expression.Quote(sql)
}
return sql + fmt.Sprintf(` and bind_sql in (%s)`, strings.Join(bindingSQLs, ","))
return sql + fmt.Sprintf(` and bind_sql = %s`, expression.Quote(bindingSQL))
}

// GenHintsFromSQL is used to generate hints from SQL.
Expand Down Expand Up @@ -703,7 +689,7 @@ func runSQL(ctx context.Context, sctx sessionctx.Context, sql string, resultChan
buf := make([]byte, 4096)
stackSize := runtime.Stack(buf, false)
buf = buf[:stackSize]
resultChan <- errors.New(fmt.Sprintf("run sql panicked: %v", string(buf)))
resultChan <- fmt.Errorf("run sql panicked: %v", string(buf))
}
}()
recordSets, err := sctx.(sqlexec.SQLExecutor).Execute(ctx, sql)
Expand Down Expand Up @@ -745,7 +731,7 @@ func (h *BindHandle) HandleEvolvePlanTask(sctx sessionctx.Context) error {
// since it is still in the bind record. Now we just drop it and if it is actually retryable,
// we will hope for that we can capture this evolve task again.
if err != nil {
return h.DropBindRecord(sctx, sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema), &BindRecord{OriginalSQL: originalSQL, Db: db, Bindings: []Binding{binding}})
return h.DropBindRecord(originalSQL, db, &binding)
}
// If the accepted plan timeouts, it is hard to decide the timeout for verify plan.
// Currently we simply mark the verify plan as `using` if it could run successfully within maxTime.
Expand All @@ -755,7 +741,7 @@ func (h *BindHandle) HandleEvolvePlanTask(sctx sessionctx.Context) error {
sctx.GetSessionVars().UsePlanBaselines = false
verifyPlanTime, err := h.getRunningDuration(sctx, db, binding.BindSQL, maxTime)
if err != nil {
return h.DropBindRecord(sctx, sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema), &BindRecord{OriginalSQL: originalSQL, Db: db, Bindings: []Binding{binding}})
return h.DropBindRecord(originalSQL, db, &binding)
}
if verifyPlanTime < 0 {
binding.Status = Rejected
Expand Down
12 changes: 6 additions & 6 deletions bindinfo/session_handle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,13 @@ func (h *SessionHandle) AddBindRecord(sctx sessionctx.Context, is infoschema.Inf
}

// DropBindRecord drops a BindRecord in the cache.
func (h *SessionHandle) DropBindRecord(sctx sessionctx.Context, is infoschema.InfoSchema, record *BindRecord) error {
err := record.prepareHints(sctx, is)
if err != nil {
return err
}
oldRecord := h.GetBindRecord(record.OriginalSQL, record.Db)
func (h *SessionHandle) DropBindRecord(originalSQL, db string, binding *Binding) error {
oldRecord := h.GetBindRecord(originalSQL, db)
var newRecord *BindRecord
record := &BindRecord{OriginalSQL: originalSQL, Db: db}
if binding != nil {
record.Bindings = append(record.Bindings, *binding)
}
if oldRecord != nil {
newRecord = oldRecord.remove(record)
} else {
Expand Down
35 changes: 18 additions & 17 deletions cmd/explaintest/r/explain_complex.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ Projection_13 1.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd
└─HashAgg_19 1.00 root group by:test.dd.dic, test.st.aid, funcs:firstrow(test.st.id)->test.st.id, funcs:firstrow(test.st.aid)->test.st.aid, funcs:firstrow(test.st.cm)->test.st.cm, funcs:firstrow(test.st.p1)->test.st.p1, funcs:firstrow(test.st.p2)->test.st.p2, funcs:firstrow(test.st.p3)->test.st.p3, funcs:firstrow(test.st.p4)->test.st.p4, funcs:firstrow(test.st.p5)->test.st.p5, funcs:firstrow(test.st.p6_md5)->test.st.p6_md5, funcs:firstrow(test.st.p7_md5)->test.st.p7_md5, funcs:firstrow(test.st.ext)->test.st.ext, funcs:firstrow(test.st.t)->test.st.t, funcs:firstrow(test.dd.id)->test.dd.id, funcs:firstrow(test.dd.dic)->test.dd.dic, funcs:firstrow(test.dd.ip)->test.dd.ip, funcs:firstrow(test.dd.t)->test.dd.t
└─IndexMergeJoin_30 0.00 root inner join, inner:IndexLookUp_28, outer key:test.st.aid, inner key:test.dd.aid, other cond:eq(test.dd.ip, test.st.ip), gt(test.dd.t, test.st.t)
├─IndexLookUp_28 0.00 root
│ ├─IndexScan_25 0.00 cop[tikv] table:dd, index:aid, dic, range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo
│ ├─IndexScan_25 1.00 cop[tikv] table:dd, index:aid, dic, range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo
│ └─Selection_27 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "android"), gt(test.dd.t, 1478143908), not(isnull(test.dd.ip)), not(isnull(test.dd.t))
│ └─TableScan_26 0.00 cop[tikv] table:dd, keep order:false, stats:pseudo
│ └─TableScan_26 1.00 cop[tikv] table:dd, keep order:false, stats:pseudo
└─IndexLookUp_41 3.33 root
├─IndexScan_38 3333.33 cop[tikv] table:gad, index:t, range:(1478143908,+inf], keep order:false, stats:pseudo
└─Selection_40 3.33 cop[tikv] eq(test.st.pt, "android"), not(isnull(test.st.ip))
Expand All @@ -137,9 +137,9 @@ Projection_10 0.00 root test.st.id, test.dd.id, test.st.aid, test.st.cm, test.dd
│ └─Selection_34 0.00 cop[tikv] eq(test.st.bm, 0), eq(test.st.dit, "mac"), eq(test.st.pt, "ios"), not(isnull(test.st.dic))
│ └─TableScan_33 3333.33 cop[tikv] table:gad, keep order:false, stats:pseudo
└─IndexLookUp_22 0.00 root
├─IndexScan_19 0.00 cop[tikv] table:sdk, index:aid, dic, range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo
├─IndexScan_19 1.00 cop[tikv] table:sdk, index:aid, dic, range: decided by [eq(test.dd.aid, test.st.aid)], keep order:true, stats:pseudo
└─Selection_21 0.00 cop[tikv] eq(test.dd.bm, 0), eq(test.dd.pt, "ios"), gt(test.dd.t, 1477971479), not(isnull(test.dd.mac)), not(isnull(test.dd.t))
└─TableScan_20 0.00 cop[tikv] table:sdk, keep order:false, stats:pseudo
└─TableScan_20 1.00 cop[tikv] table:sdk, keep order:false, stats:pseudo
explain SELECT cm, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(1) as click_pv, count(DISTINCT ip) as click_ip FROM st WHERE (t between 1478188800 and 1478275200) and aid='cn.sbkcq' and pt='android' GROUP BY cm, p1, p2, p3, p4, p5, p6_md5, p7_md5;
id count task operator info
Projection_5 1.00 root test.st.cm, test.st.p1, test.st.p2, test.st.p3, test.st.p4, test.st.p5, test.st.p6_md5, test.st.p7_md5, Column#20, Column#21
Expand All @@ -152,14 +152,15 @@ explain select dt.id as id, dt.aid as aid, dt.pt as pt, dt.dic as dic, dt.cm as
id count task operator info
Projection_10 0.00 root test.dt.id, test.dt.aid, test.dt.pt, test.dt.dic, test.dt.cm, test.rr.gid, test.rr.acd, test.rr.t, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5
└─Limit_13 0.00 root offset:0, count:2000
└─IndexJoin_19 0.00 root inner join, inner:IndexLookUp_18, outer key:test.dt.aid, test.dt.dic, inner key:test.rr.aid, test.rr.dic
├─TableReader_43 0.00 root data:Selection_42
│ └─Selection_42 0.00 cop[tikv] eq(test.dt.bm, 0), eq(test.dt.pt, "ios"), gt(test.dt.t, 1478185592), not(isnull(test.dt.dic))
│ └─TableScan_41 10000.00 cop[tikv] table:dt, range:[0,+inf], keep order:false, stats:pseudo
└─IndexLookUp_18 0.00 root
├─IndexScan_15 1.25 cop[tikv] table:rr, index:aid, dic, range: decided by [eq(test.rr.aid, test.dt.aid) eq(test.rr.dic, test.dt.dic)], keep order:false, stats:pseudo
└─Selection_17 0.00 cop[tikv] eq(test.rr.pt, "ios"), gt(test.rr.t, 1478185592)
└─TableScan_16 1.25 cop[tikv] table:rr, keep order:false, stats:pseudo
└─IndexMergeJoin_41 0.00 root inner join, inner:IndexLookUp_39, outer key:test.rr.aid, test.rr.dic, inner key:test.dt.aid, test.dt.dic
├─IndexLookUp_39 0.00 root
│ ├─Selection_37 1.00 cop[tikv] not(isnull(test.dt.dic))
│ │ └─IndexScan_35 1.00 cop[tikv] table:dt, index:aid, dic, range: decided by [eq(test.dt.aid, test.rr.aid) eq(test.dt.dic, test.rr.dic)], keep order:true, stats:pseudo
│ └─Selection_38 0.00 cop[tikv] eq(test.dt.bm, 0), eq(test.dt.pt, "ios"), gt(test.dt.t, 1478185592)
│ └─TableScan_36 1.00 cop[tikv] table:dt, keep order:false, stats:pseudo
└─TableReader_61 3.33 root data:Selection_60
└─Selection_60 3.33 cop[tikv] eq(test.rr.pt, "ios"), gt(test.rr.t, 1478185592)
└─TableScan_59 10000.00 cop[tikv] table:rr, range:[-inf,+inf], keep order:false, stats:pseudo
explain select pc,cr,count(DISTINCT uid) as pay_users,count(oid) as pay_times,sum(am) as am from pp where ps=2 and ppt>=1478188800 and ppt<1478275200 and pi in ('510017','520017') and uid in ('18089709','18090780') group by pc,cr;
id count task operator info
Projection_5 1.00 root test.pp.pc, test.pp.cr, Column#22, Column#23, Column#24
Expand Down Expand Up @@ -251,11 +252,11 @@ Sort_10 1.00 root test.org_department.left_value:asc
│ │ ├─IndexScan_59 10.00 cop[tikv] table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo
│ │ └─Selection_61 0.01 cop[tikv] eq(test.org_department.status, 1000)
│ │ └─TableScan_60 10.00 cop[tikv] table:d, keep order:false, stats:pseudo
│ └─IndexLookUp_38 0.00 root
│ ├─Selection_36 1.25 cop[tikv] not(isnull(test.org_position.department_id))
│ │ └─IndexScan_34 1.25 cop[tikv] table:p, index:department_id, range: decided by [eq(test.org_position.department_id, test.org_department.id)], keep order:true, stats:pseudo
│ └─Selection_37 0.00 cop[tikv] eq(test.org_position.status, 1000)
│ └─TableScan_35 1.25 cop[tikv] table:p, keep order:false, stats:pseudo
│ └─IndexLookUp_38 1.25 root
│ ├─Selection_36 1250.00 cop[tikv] not(isnull(test.org_position.department_id))
│ │ └─IndexScan_34 1251.25 cop[tikv] table:p, index:department_id, range: decided by [eq(test.org_position.department_id, test.org_department.id)], keep order:true, stats:pseudo
│ └─Selection_37 1.25 cop[tikv] eq(test.org_position.status, 1000)
│ └─TableScan_35 1250.00 cop[tikv] table:p, keep order:false, stats:pseudo
└─TableReader_72 9.99 root data:Selection_71
└─Selection_71 9.99 cop[tikv] eq(test.org_employee_position.status, 1000), not(isnull(test.org_employee_position.position_id))
└─TableScan_70 10000.00 cop[tikv] table:ep, range:[-inf,+inf], keep order:false, stats:pseudo
Loading

0 comments on commit fb547ba

Please sign in to comment.