From c16b3f65b3a7cacf284bd82e1f62e31499a5a6f3 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 21 Jun 2023 11:14:41 +0800 Subject: [PATCH] planner: plan cache supports queries with more than 200 parameters (#44829) close pingcap/tidb#44823 --- planner/core/plan_cache_test.go | 27 +++++----- planner/core/plan_cacheable_checker.go | 49 +++++++++++++---- planner/core/plan_cacheable_checker_test.go | 58 +++++++++++++++++++++ sessionctx/variable/session.go | 2 + 4 files changed, 113 insertions(+), 23 deletions(-) diff --git a/planner/core/plan_cache_test.go b/planner/core/plan_cache_test.go index 555bd80d620de..a476239815535 100644 --- a/planner/core/plan_cache_test.go +++ b/planner/core/plan_cache_test.go @@ -638,24 +638,25 @@ func TestPreparedPlanCacheLongInList(t *testing.T) { return "(" + strings.Join(elements, ",") + ")" } - tk.MustExec(fmt.Sprintf(`prepare st_99 from 'select * from t where a in %v'`, genInList(99))) - tk.MustExec(`execute st_99`) - tk.MustExec(`execute st_99`) + // the limitation is 200 + tk.MustExec(fmt.Sprintf(`prepare st_199 from 'select * from t where a in %v'`, genInList(199))) + tk.MustExec(`execute st_199`) + tk.MustExec(`execute st_199`) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) - tk.MustExec(fmt.Sprintf(`prepare st_101 from 'select * from t where a in %v'`, genInList(101))) - tk.MustExec(`execute st_101`) - tk.MustExec(`execute st_101`) + tk.MustExec(fmt.Sprintf(`prepare st_201 from 'select * from t where a in %v'`, genInList(201))) + tk.MustExec(`execute st_201`) + tk.MustExec(`execute st_201`) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) - tk.MustExec(fmt.Sprintf(`prepare st_49_50 from 'select * from t where a in %v and b in %v'`, genInList(49), genInList(50))) - tk.MustExec(`execute st_49_50`) - tk.MustExec(`execute st_49_50`) + tk.MustExec(fmt.Sprintf(`prepare st_99_100 from 'select * from t where a in %v and b in %v'`, genInList(99), genInList(100))) + tk.MustExec(`execute st_99_100`) + tk.MustExec(`execute st_99_100`) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) - tk.MustExec(fmt.Sprintf(`prepare st_49_52 from 'select * from t where a in %v and b in %v'`, genInList(49), genInList(52))) - tk.MustExec(`execute st_49_52`) - tk.MustExec(`execute st_49_52`) + tk.MustExec(fmt.Sprintf(`prepare st_100_101 from 'select * from t where a in %v and b in %v'`, genInList(100), genInList(101))) + tk.MustExec(`execute st_100_101`) + tk.MustExec(`execute st_100_101`) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) } @@ -1210,7 +1211,7 @@ func TestLongInsertStmt(t *testing.T) { tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) tk.MustExec(`prepare inert201 from 'insert into t values (1)` + strings.Repeat(", (1)", 200) + "'") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip prepared plan-cache: too many values (more than 200) in the insert statement")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 skip prepared plan-cache: too many values in the insert statement")) tk.MustExec(`execute inert201`) tk.MustExec(`execute inert201`) tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) diff --git a/planner/core/plan_cacheable_checker.go b/planner/core/plan_cacheable_checker.go index 85496ce304138..1246df1473c84 100644 --- a/planner/core/plan_cacheable_checker.go +++ b/planner/core/plan_cacheable_checker.go @@ -16,6 +16,8 @@ package core import ( "fmt" + "math" + "strconv" "sync" "github.com/pingcap/tidb/expression" @@ -26,6 +28,7 @@ import ( "github.com/pingcap/tidb/parser/mysql" core_metrics "github.com/pingcap/tidb/planner/core/metrics" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/filter" @@ -56,6 +59,7 @@ func CacheableWithCtx(sctx sessionctx.Context, node ast.Node, is infoschema.Info cacheable: true, schema: is, sumInListLen: 0, + maxNumParam: getMaxParamLimit(sctx), } node.Accept(&checker) return checker.cacheable, checker.reason @@ -69,6 +73,7 @@ type cacheableChecker struct { reason string // reason why cannot use plan-cache sumInListLen int // the accumulated number of elements in all in-lists + maxNumParam int } // Enter implements Visitor interface. @@ -105,9 +110,9 @@ func (checker *cacheableChecker) Enter(in ast.Node) (out ast.Node, skipChildren if len(node.Lists) > 0 { // avoid index-out-of-range nCols = len(node.Lists[0]) } - if nRows*nCols > 200 { // to save memory + if nRows*nCols > checker.maxNumParam { // to save memory checker.cacheable = false - checker.reason = "too many values (more than 200) in the insert statement" + checker.reason = "too many values in the insert statement" return in, true } } @@ -120,9 +125,9 @@ func (checker *cacheableChecker) Enter(in ast.Node) (out ast.Node, skipChildren } case *ast.PatternInExpr: checker.sumInListLen += len(node.List) - if checker.sumInListLen > 100 { // to save memory + if checker.sumInListLen > checker.maxNumParam { // to save memory checker.cacheable = false - checker.reason = "too many values in in-list (more than 100)" + checker.reason = "too many values in in-list" return in, true } case *ast.VariableExpr: @@ -223,6 +228,7 @@ func NonPreparedPlanCacheableWithCtx(sctx sessionctx.Context, node ast.Node, is return false, "not a SELECT statement" } + maxNumParam := getMaxParamLimit(sctx) var tableNames []*ast.TableName switch x := node.(type) { case *ast.SelectStmt: @@ -251,8 +257,8 @@ func NonPreparedPlanCacheableWithCtx(sctx sessionctx.Context, node ast.Node, is if len(x.Lists) > 0 { // avoid index-out-of-range nCols = len(x.Lists[0]) } - if nRows*nCols > 200 { // to save memory - return false, "too many values (more than 200) in the insert statement" + if nRows*nCols > maxNumParam { // to save memory + return false, "too many values in the insert statement" } tableNames, ok, reason = extractTableNames(x.Table.TableRefs, tableNames) if !ok { @@ -289,7 +295,7 @@ func NonPreparedPlanCacheableWithCtx(sctx sessionctx.Context, node ast.Node, is // allocate and init the checker checker := nonPrepCacheCheckerPool.Get().(*nonPreparedPlanCacheableChecker) - checker.reset(sctx, is, tableNames) + checker.reset(sctx, is, tableNames, maxNumParam) node.Accept(checker) cacheable, reason := checker.cacheable, checker.reason @@ -382,9 +388,11 @@ type nonPreparedPlanCacheableChecker struct { constCnt int // the number of constants/parameters in this query filterCnt int // the number of filters in the current node + + maxNumberParam int // the maximum number of parameters for a query to be cached. } -func (checker *nonPreparedPlanCacheableChecker) reset(sctx sessionctx.Context, schema infoschema.InfoSchema, tableNodes []*ast.TableName) { +func (checker *nonPreparedPlanCacheableChecker) reset(sctx sessionctx.Context, schema infoschema.InfoSchema, tableNodes []*ast.TableName, maxNumberParam int) { checker.sctx = sctx checker.cacheable = true checker.schema = schema @@ -392,6 +400,7 @@ func (checker *nonPreparedPlanCacheableChecker) reset(sctx sessionctx.Context, s checker.tableNodes = tableNodes checker.constCnt = 0 checker.filterCnt = 0 + checker.maxNumberParam = maxNumberParam } // Enter implements Visitor interface. @@ -458,9 +467,9 @@ func (checker *nonPreparedPlanCacheableChecker) Enter(in ast.Node) (out ast.Node checker.reason = "query has null constants" } checker.constCnt++ - if checker.constCnt > 200 { // just for safety and reduce memory cost + if checker.maxNumberParam > 0 && checker.constCnt > checker.maxNumberParam { // just for safety and reduce memory cost checker.cacheable = false - checker.reason = "query has more than 200 constants" + checker.reason = "query has too many constants" } return in, !checker.cacheable case *ast.GroupByClause: @@ -672,3 +681,23 @@ func isPhysicalPlanCacheable(sctx sessionctx.Context, p PhysicalPlan, paramNum, } return true, "" } + +// getMaxParamLimit returns the maximum number of parameters for a query that can be cached in the Plan Cache. +func getMaxParamLimit(sctx sessionctx.Context) int { + v := 200 + if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().OptimizerFixControl == nil { + return v + } + if sctx.GetSessionVars().OptimizerFixControl[variable.TiDBOptFixControl44823] != "" { + n, err := strconv.Atoi(sctx.GetSessionVars().OptimizerFixControl[variable.TiDBOptFixControl44823]) + if err != nil { + return v + } + if n == 0 { + v = math.MaxInt32 // no limitation + } else if n > 0 { + v = n + } + } + return v +} diff --git a/planner/core/plan_cacheable_checker_test.go b/planner/core/plan_cacheable_checker_test.go index fae602e5f42cb..70ff91872d36d 100644 --- a/planner/core/plan_cacheable_checker_test.go +++ b/planner/core/plan_cacheable_checker_test.go @@ -15,6 +15,8 @@ package core_test import ( + "fmt" + "strings" "testing" "github.com/pingcap/tidb/expression" @@ -31,6 +33,62 @@ import ( "github.com/stretchr/testify/require" ) +func TestFixControl44823(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec(`create table t (a int)`) + var va []string + for i := 0; i < 201; i++ { + tk.MustExec(fmt.Sprintf(`set @a%v = %v`, i, i)) + va = append(va, fmt.Sprintf("@a%v", i)) + } + + // prepared plan cache + tk.MustExec(fmt.Sprintf(`prepare st from 'select * from t where a in (%v?)'`, strings.Repeat("?,", 200))) + tk.MustQuery(`show warnings`).Check(testkit.Rows(`Warning 1105 skip prepared plan-cache: too many values in in-list`)) + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) + + tk.MustExec(`set @@tidb_opt_fix_control = "44823:250"`) + tk.MustExec(fmt.Sprintf(`prepare st from 'select * from t where a in (%v?)'`, strings.Repeat("?,", 200))) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) // can hit + + tk.MustExec(`set @@tidb_opt_fix_control = "44823:0"`) + tk.MustExec(fmt.Sprintf(`prepare st from 'select * from t where a in (%v?)'`, strings.Repeat("?,", 200))) + tk.MustQuery(`show warnings`).Check(testkit.Rows()) // no warning + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustExec(fmt.Sprintf(`execute st using %v`, strings.Join(va, ","))) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + + // non prepared plan cache + values := make([]string, 0, 201) + for i := 0; i < 201; i++ { + values = append(values, fmt.Sprintf("%v", i)) + } + query := fmt.Sprintf("select * from t where a in (%v)", strings.Join(values, ",")) + tk.MustExec(`set tidb_enable_non_prepared_plan_cache=1`) + + tk.MustExec(`set @@tidb_opt_fix_control = ""`) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("0")) + + tk.MustExec(`set @@tidb_opt_fix_control = "44823:250"`) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) + + tk.MustExec(`set @@tidb_opt_fix_control = "44823:0"`) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(query).Check(testkit.Rows()) + tk.MustQuery(`select @@last_plan_from_cache`).Check(testkit.Rows("1")) +} + func TestCacheable(t *testing.T) { store := testkit.CreateMockStore(t) mockCtx := mock.NewContext() diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 1918e1e63a580..3fe5d0a7d71aa 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1518,6 +1518,8 @@ var ( TiDBOptFixControl44262 uint64 = 44262 // TiDBOptFixControl44389 controls whether to consider non-point ranges of some CNF item when building ranges. TiDBOptFixControl44389 uint64 = 44389 + // TiDBOptFixControl44823 controls the maximum number of parameters for a query that can be cached in the Plan Cache. + TiDBOptFixControl44823 uint64 = 44823 ) // GetOptimizerFixControlValue returns the specified value of the optimizer fix control.