Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

session, planner: Add a factor getter for networkFactor/scanFactor/descScanFactor/seekFactor #25046

Merged
merged 24 commits into from
Jun 3, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions planner/core/exhaust_physical_plans.go
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,7 @@ func (p *LogicalJoin) constructInnerTableScanTask(
copTask := &copTask{
tablePlan: ts,
indexPlanFinished: true,
cst: sessVars.ScanFactor * rowSize * ts.stats.RowCount,
cst: sessVars.GetScanFactor(ts.Table) * rowSize * ts.stats.RowCount,
tblColHists: ds.TblColHists,
keepOrder: ts.KeepOrder,
}
Expand Down Expand Up @@ -1088,7 +1088,7 @@ func (p *LogicalJoin) constructInnerIndexScanTask(
is.stats = ds.tableStats.ScaleByExpectCnt(tmpPath.CountAfterAccess)
rowSize := is.indexScanRowSize(path.Index, ds, true)
sessVars := ds.ctx.GetSessionVars()
cop.cst = tmpPath.CountAfterAccess * rowSize * sessVars.ScanFactor
cop.cst = tmpPath.CountAfterAccess * rowSize * sessVars.GetScanFactor(ds.tableInfo)
finalStats := ds.tableStats.ScaleByExpectCnt(rowCount)
is.addPushedDownSelection(cop, ds, tmpPath, finalStats)
t := cop.convertToRootTask(ds.ctx)
Expand Down
26 changes: 13 additions & 13 deletions planner/core/find_best_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -880,10 +880,10 @@ func (ds *DataSource) convertToPartialIndexScan(prop *property.PhysicalProperty,
}
indexPlan := PhysicalSelection{Conditions: indexConds}.Init(is.ctx, stats, ds.blockOffset)
indexPlan.SetChildren(is)
partialCost += rowCount * rowSize * sessVars.NetworkFactor
partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo)
return indexPlan, partialCost
}
partialCost += rowCount * rowSize * sessVars.NetworkFactor
partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo)
indexPlan = is
return indexPlan, partialCost
}
Expand All @@ -903,10 +903,10 @@ func (ds *DataSource) convertToPartialTableScan(prop *property.PhysicalProperty,
tablePlan = PhysicalSelection{Conditions: ts.filterCondition}.Init(ts.ctx, ts.stats.ScaleByExpectCnt(selectivity*rowCount), ds.blockOffset)
tablePlan.SetChildren(ts)
partialCost += rowCount * sessVars.CopCPUFactor
partialCost += selectivity * rowCount * rowSize * sessVars.NetworkFactor
partialCost += selectivity * rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo)
return tablePlan, partialCost
}
partialCost += rowCount * rowSize * sessVars.NetworkFactor
partialCost += rowCount * rowSize * sessVars.GetNetworkFactor(ds.tableInfo)
tablePlan = ts
return tablePlan, partialCost
}
Expand Down Expand Up @@ -971,7 +971,7 @@ func (ds *DataSource) buildIndexMergeTableScan(prop *property.PhysicalProperty,
}
}
rowSize := ds.TblColHists.GetTableAvgRowSize(ds.ctx, ds.TblCols, ts.StoreType, true)
partialCost += totalRowCount * rowSize * sessVars.ScanFactor
partialCost += totalRowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo)
ts.stats = ds.tableStats.ScaleByExpectCnt(totalRowCount)
if ds.statisticTable.Pseudo {
ts.stats.StatsVersion = statistics.PseudoVersion
Expand Down Expand Up @@ -1873,22 +1873,22 @@ func (ds *DataSource) getOriginalPhysicalTableScan(prop *property.PhysicalProper
rowSize = ds.TblColHists.GetTableAvgRowSize(ds.ctx, ts.Schema().Columns, ts.StoreType, ds.handleCols != nil)
}
sessVars := ds.ctx.GetSessionVars()
cost := rowCount * rowSize * sessVars.ScanFactor
cost := rowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo)
if ts.IsGlobalRead {
cost += rowCount * sessVars.NetworkFactor * rowSize
cost += rowCount * sessVars.GetNetworkFactor(ds.tableInfo) * rowSize
}
if isMatchProp {
ts.Desc = prop.SortItems[0].Desc
if prop.SortItems[0].Desc && prop.ExpectedCnt >= smallScanThreshold {
cost = rowCount * rowSize * sessVars.DescScanFactor
cost = rowCount * rowSize * sessVars.GetDescScanFactor(ds.tableInfo)
}
ts.KeepOrder = true
}
switch ts.StoreType {
case kv.TiKV:
cost += float64(len(ts.Ranges)) * sessVars.SeekFactor
cost += float64(len(ts.Ranges)) * sessVars.GetSeekFactor(ds.tableInfo)
case kv.TiFlash:
cost += float64(len(ts.Ranges)) * float64(len(ts.Columns)) * sessVars.SeekFactor
cost += float64(len(ts.Ranges)) * float64(len(ts.Columns)) * sessVars.GetSeekFactor(ds.tableInfo)
}
return ts, cost, rowCount
}
Expand Down Expand Up @@ -1928,15 +1928,15 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper
is.stats = ds.tableStats.ScaleByExpectCnt(rowCount)
rowSize := is.indexScanRowSize(idx, ds, true)
sessVars := ds.ctx.GetSessionVars()
cost := rowCount * rowSize * sessVars.ScanFactor
cost := rowCount * rowSize * sessVars.GetScanFactor(ds.tableInfo)
if isMatchProp {
is.Desc = prop.SortItems[0].Desc
if prop.SortItems[0].Desc && prop.ExpectedCnt >= smallScanThreshold {
cost = rowCount * rowSize * sessVars.DescScanFactor
cost = rowCount * rowSize * sessVars.GetDescScanFactor(ds.tableInfo)
}
is.KeepOrder = true
}
cost += float64(len(is.Ranges)) * sessVars.SeekFactor
cost += float64(len(is.Ranges)) * sessVars.GetSeekFactor(ds.tableInfo)
is.cost = cost
return is, cost, rowCount
}
Expand Down
8 changes: 4 additions & 4 deletions planner/core/point_get_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,8 @@ func (p *PointGetPlan) GetCost(cols []*expression.Column) float64 {
} else {
rowSize = p.stats.HistColl.GetIndexAvgRowSize(p.ctx, cols, p.IndexInfo.Unique)
}
cost += rowSize * sessVars.NetworkFactor
cost += sessVars.SeekFactor
cost += rowSize * sessVars.GetNetworkFactor(p.TblInfo)
cost += sessVars.GetSeekFactor(p.TblInfo)
cost /= float64(sessVars.DistSQLScanConcurrency())
return cost
}
Expand Down Expand Up @@ -428,8 +428,8 @@ func (p *BatchPointGetPlan) GetCost(cols []*expression.Column) float64 {
rowCount = float64(len(p.IndexValues))
rowSize = p.stats.HistColl.GetIndexAvgRowSize(p.ctx, cols, p.IndexInfo.Unique)
}
cost += rowCount * rowSize * sessVars.NetworkFactor
cost += rowCount * sessVars.SeekFactor
cost += rowCount * rowSize * sessVars.GetNetworkFactor(p.TblInfo)
cost += rowCount * sessVars.GetSeekFactor(p.TblInfo)
cost /= float64(sessVars.DistSQLScanConcurrency())
return cost
}
Expand Down
18 changes: 12 additions & 6 deletions planner/core/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/charset"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/expression"
Expand Down Expand Up @@ -150,19 +151,24 @@ func (t *copTask) finishIndexPlan() {
cnt := t.count()
t.indexPlanFinished = true
sessVars := t.indexPlan.SCtx().GetSessionVars()
var tableInfo *model.TableInfo
if t.tablePlan != nil {
ts := t.tablePlan.(*PhysicalTableScan)
ts.stats = t.indexPlan.statsInfo()
tableInfo = ts.Table
}
// Network cost of transferring rows of index scan to TiDB.
t.cst += cnt * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(t.indexPlan.SCtx(), t.indexPlan.Schema().Columns, true, false)

t.cst += cnt * sessVars.GetNetworkFactor(tableInfo) * t.tblColHists.GetAvgRowSize(t.indexPlan.SCtx(), t.indexPlan.Schema().Columns, true, false)
if t.tablePlan == nil {
return
}

// Calculate the IO cost of table scan here because we cannot know its stats until we finish index plan.
t.tablePlan.(*PhysicalTableScan).stats = t.indexPlan.statsInfo()
var p PhysicalPlan
for p = t.indexPlan; len(p.Children()) > 0; p = p.Children()[0] {
}
rowSize := t.tblColHists.GetIndexAvgRowSize(t.indexPlan.SCtx(), t.tblCols, p.(*PhysicalIndexScan).Index.Unique)
t.cst += cnt * rowSize * sessVars.ScanFactor
t.cst += cnt * rowSize * sessVars.GetScanFactor(tableInfo)
}

func (t *copTask) getStoreType() kv.StoreType {
Expand Down Expand Up @@ -935,7 +941,7 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask {
var prevSchema *expression.Schema
// Network cost of transferring rows of table scan to TiDB.
if t.tablePlan != nil {
t.cst += t.count() * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false)
t.cst += t.count() * sessVars.GetNetworkFactor(nil) * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false)

tp := t.tablePlan
for len(tp.Children()) > 0 {
Expand Down Expand Up @@ -2111,7 +2117,7 @@ func (t *mppTask) enforceExchangerImpl(prop *property.PhysicalProperty) *mppTask
sender.SetChildren(t.p)
receiver := PhysicalExchangeReceiver{}.Init(ctx, t.p.statsInfo())
receiver.SetChildren(sender)
cst := t.cst + t.count()*ctx.GetSessionVars().NetworkFactor
cst := t.cst + t.count()*ctx.GetSessionVars().GetNetworkFactor(nil)
sender.cost = cst
receiver.cost = cst
return &mppTask{
Expand Down
16 changes: 9 additions & 7 deletions planner/implementation/datasource.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ func (impl *TableReaderImpl) CalcCost(outCount float64, children ...memo.Impleme
reader := impl.plan.(*plannercore.PhysicalTableReader)
width := impl.tblColHists.GetAvgRowSize(impl.plan.SCtx(), reader.Schema().Columns, false, false)
sessVars := reader.SCtx().GetSessionVars()
networkCost := outCount * sessVars.NetworkFactor * width
// TableReaderImpl don't have tableInfo property, so using nil to replace it.
// Todo add the tableInfo property for the TableReaderImpl.
networkCost := outCount * sessVars.GetNetworkFactor(nil) * width
mmyj marked this conversation as resolved.
Show resolved Hide resolved
// copTasks are run in parallel, to make the estimated cost closer to execution time, we amortize
// the cost to cop iterator workers. According to `CopClient::Send`, the concurrency
// is Min(DistSQLScanConcurrency, numRegionsInvolvedInScan), since we cannot infer
Expand Down Expand Up @@ -118,9 +120,9 @@ func (impl *TableScanImpl) CalcCost(outCount float64, children ...memo.Implement
ts := impl.plan.(*plannercore.PhysicalTableScan)
width := impl.tblColHists.GetTableAvgRowSize(impl.plan.SCtx(), impl.tblCols, kv.TiKV, true)
sessVars := ts.SCtx().GetSessionVars()
impl.cost = outCount * sessVars.ScanFactor * width
impl.cost = outCount * sessVars.GetScanFactor(ts.Table) * width
if ts.Desc {
impl.cost = outCount * sessVars.DescScanFactor * width
impl.cost = outCount * sessVars.GetDescScanFactor(ts.Table) * width
}
return impl.cost
}
Expand All @@ -146,7 +148,7 @@ func (impl *IndexReaderImpl) GetCostLimit(costLimit float64, children ...memo.Im
func (impl *IndexReaderImpl) CalcCost(outCount float64, children ...memo.Implementation) float64 {
reader := impl.plan.(*plannercore.PhysicalIndexReader)
sessVars := reader.SCtx().GetSessionVars()
networkCost := outCount * sessVars.NetworkFactor * impl.tblColHists.GetAvgRowSize(reader.SCtx(), children[0].GetPlan().Schema().Columns, true, false)
networkCost := outCount * sessVars.GetNetworkFactor(nil) * impl.tblColHists.GetAvgRowSize(reader.SCtx(), children[0].GetPlan().Schema().Columns, true, false)
copIterWorkers := float64(sessVars.DistSQLScanConcurrency())
impl.cost = (networkCost + children[0].GetCost()) / copIterWorkers
return impl.cost
Expand All @@ -171,11 +173,11 @@ func (impl *IndexScanImpl) CalcCost(outCount float64, children ...memo.Implement
is := impl.plan.(*plannercore.PhysicalIndexScan)
sessVars := is.SCtx().GetSessionVars()
rowSize := impl.tblColHists.GetIndexAvgRowSize(is.SCtx(), is.Schema().Columns, is.Index.Unique)
cost := outCount * rowSize * sessVars.ScanFactor
cost := outCount * rowSize * sessVars.GetScanFactor(is.Table)
if is.Desc {
cost = outCount * rowSize * sessVars.DescScanFactor
cost = outCount * rowSize * sessVars.GetDescScanFactor(is.Table)
}
cost += float64(len(is.Ranges)) * sessVars.SeekFactor
cost += float64(len(is.Ranges)) * sessVars.GetSeekFactor(is.Table)
impl.cost = cost
return impl.cost
}
Expand Down
66 changes: 55 additions & 11 deletions sessionctx/variable/session.go
Original file line number Diff line number Diff line change
Expand Up @@ -526,14 +526,14 @@ type SessionVars struct {
CopCPUFactor float64
// CopTiFlashConcurrencyFactor is the concurrency number of computation in tiflash coprocessor.
CopTiFlashConcurrencyFactor float64
// NetworkFactor is the network cost of transferring 1 byte data.
NetworkFactor float64
// networkFactor is the network cost of transferring 1 byte data.
networkFactor float64
// ScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash.
ScanFactor float64
// DescScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash in desc order.
DescScanFactor float64
// SeekFactor is the IO cost of seeking the start value of a range in TiKV or TiFlash.
SeekFactor float64
scanFactor float64
// descScanFactor is the IO cost of scanning 1 byte data on TiKV and TiFlash in desc order.
descScanFactor float64
// seekFactor is the IO cost of seeking the start value of a range in TiKV or TiFlash.
seekFactor float64
// MemoryFactor is the memory cost of storing one tuple.
MemoryFactor float64
// DiskFactor is the IO cost of reading/writing one byte to temporary disk.
Expand Down Expand Up @@ -984,10 +984,10 @@ func NewSessionVars() *SessionVars {
CPUFactor: DefOptCPUFactor,
CopCPUFactor: DefOptCopCPUFactor,
CopTiFlashConcurrencyFactor: DefOptTiFlashConcurrencyFactor,
NetworkFactor: DefOptNetworkFactor,
ScanFactor: DefOptScanFactor,
DescScanFactor: DefOptDescScanFactor,
SeekFactor: DefOptSeekFactor,
networkFactor: DefOptNetworkFactor,
scanFactor: DefOptScanFactor,
descScanFactor: DefOptDescScanFactor,
seekFactor: DefOptSeekFactor,
MemoryFactor: DefOptMemoryFactor,
DiskFactor: DefOptDiskFactor,
ConcurrencyFactor: DefOptConcurrencyFactor,
Expand Down Expand Up @@ -2113,3 +2113,47 @@ func (s *SessionVars) CleanupTxnReadTSIfUsed() {
s.SnapshotInfoschema = nil
}
}

// GetNetworkFactor returns the session variable networkFactor
// returns 0 when tbl is a temporary table.
func (s *SessionVars) GetNetworkFactor(tbl *model.TableInfo) float64 {
if tbl != nil {
if tbl.TempTableType != model.TempTableNone {
return 0
}
}
return s.networkFactor
}

// GetScanFactor returns the session variable scanFactor
// returns 0 when tbl is a temporary table.
func (s *SessionVars) GetScanFactor(tbl *model.TableInfo) float64 {
if tbl != nil {
if tbl.TempTableType != model.TempTableNone {
return 0
}
}
return s.scanFactor
}

// GetDescScanFactor returns the session variable descScanFactor
// returns 0 when tbl is a temporary table.
func (s *SessionVars) GetDescScanFactor(tbl *model.TableInfo) float64 {
if tbl != nil {
if tbl.TempTableType != model.TempTableNone {
return 0
}
}
return s.descScanFactor
}

// GetSeekFactor returns the session variable seekFactor
// returns 0 when tbl is a temporary table.
func (s *SessionVars) GetSeekFactor(tbl *model.TableInfo) float64 {
if tbl != nil {
if tbl.TempTableType != model.TempTableNone {
return 0
}
}
return s.seekFactor
}
8 changes: 4 additions & 4 deletions sessionctx/variable/sysvar.go
Original file line number Diff line number Diff line change
Expand Up @@ -918,19 +918,19 @@ var defaultSysVars = []*SysVar{
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptNetworkFactor, Value: strconv.FormatFloat(DefOptNetworkFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
s.NetworkFactor = tidbOptFloat64(val, DefOptNetworkFactor)
s.networkFactor = tidbOptFloat64(val, DefOptNetworkFactor)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptScanFactor, Value: strconv.FormatFloat(DefOptScanFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
s.ScanFactor = tidbOptFloat64(val, DefOptScanFactor)
s.scanFactor = tidbOptFloat64(val, DefOptScanFactor)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptDescScanFactor, Value: strconv.FormatFloat(DefOptDescScanFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
s.DescScanFactor = tidbOptFloat64(val, DefOptDescScanFactor)
s.descScanFactor = tidbOptFloat64(val, DefOptDescScanFactor)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptSeekFactor, Value: strconv.FormatFloat(DefOptSeekFactor, 'f', -1, 64), skipInit: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
s.SeekFactor = tidbOptFloat64(val, DefOptSeekFactor)
s.seekFactor = tidbOptFloat64(val, DefOptSeekFactor)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptMemoryFactor, Value: strconv.FormatFloat(DefOptMemoryFactor, 'f', -1, 64), Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error {
Expand Down
16 changes: 8 additions & 8 deletions sessionctx/variable/varsutil_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,37 +369,37 @@ func (s *testVarsutilSuite) TestVarsutil(c *C) {
c.Assert(val, Equals, "5.0")
c.Assert(v.CopCPUFactor, Equals, 5.0)

c.Assert(v.NetworkFactor, Equals, 1.0)
c.Assert(v.GetNetworkFactor(nil), Equals, 1.0)
err = SetSessionSystemVar(v, TiDBOptNetworkFactor, "3.0")
c.Assert(err, IsNil)
val, err = GetSessionOrGlobalSystemVar(v, TiDBOptNetworkFactor)
c.Assert(err, IsNil)
c.Assert(val, Equals, "3.0")
c.Assert(v.NetworkFactor, Equals, 3.0)
c.Assert(v.GetNetworkFactor(nil), Equals, 3.0)

c.Assert(v.ScanFactor, Equals, 1.5)
c.Assert(v.GetScanFactor(nil), Equals, 1.5)
err = SetSessionSystemVar(v, TiDBOptScanFactor, "3.0")
c.Assert(err, IsNil)
val, err = GetSessionOrGlobalSystemVar(v, TiDBOptScanFactor)
c.Assert(err, IsNil)
c.Assert(val, Equals, "3.0")
c.Assert(v.ScanFactor, Equals, 3.0)
c.Assert(v.GetScanFactor(nil), Equals, 3.0)

c.Assert(v.DescScanFactor, Equals, 3.0)
c.Assert(v.GetDescScanFactor(nil), Equals, 3.0)
err = SetSessionSystemVar(v, TiDBOptDescScanFactor, "5.0")
c.Assert(err, IsNil)
val, err = GetSessionOrGlobalSystemVar(v, TiDBOptDescScanFactor)
c.Assert(err, IsNil)
c.Assert(val, Equals, "5.0")
c.Assert(v.DescScanFactor, Equals, 5.0)
c.Assert(v.GetDescScanFactor(nil), Equals, 5.0)

c.Assert(v.SeekFactor, Equals, 20.0)
c.Assert(v.GetSeekFactor(nil), Equals, 20.0)
err = SetSessionSystemVar(v, TiDBOptSeekFactor, "50.0")
c.Assert(err, IsNil)
val, err = GetSessionOrGlobalSystemVar(v, TiDBOptSeekFactor)
c.Assert(err, IsNil)
c.Assert(val, Equals, "50.0")
c.Assert(v.SeekFactor, Equals, 50.0)
c.Assert(v.GetSeekFactor(nil), Equals, 50.0)

c.Assert(v.MemoryFactor, Equals, 0.001)
err = SetSessionSystemVar(v, TiDBOptMemoryFactor, "1.0")
Expand Down