Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

session: add a batch commit session variable for the large transaction #8293

Merged
merged 16 commits into from
Dec 10, 2018
Merged

session: add a batch commit session variable for the large transaction #8293

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
1b0d658
support batch commit for the TiDB server
jackysp Nov 13, 2018
46dcb7c
tiny fix
jackysp Nov 13, 2018
6c020f6
address comments
jackysp Nov 13, 2018
c326322
add test cases
jackysp Nov 15, 2018
740387d
tiny fix
jackysp Nov 15, 2018
fab5f83
Merge branch 'master' into batch_commit
jackysp Nov 16, 2018
16a645d
add one case
jackysp Nov 16, 2018
809dc75
Merge branch 'master' into batch_commit
jackysp Nov 27, 2018
8982dd2
move to a session var
jackysp Dec 3, 2018
dec9ba1
fix test
jackysp Dec 3, 2018
8cbcd37
fix CI
jackysp Dec 3, 2018
72c1139
add a set test
jackysp Dec 3, 2018
f06c961
Merge branch 'master' into batch_commit
lysu Dec 4, 2018
7982488
Merge branch 'master' into batch_commit
jackysp Dec 10, 2018
7ef4275
resolve conflicts
jackysp Dec 10, 2018
8128841
Merge branch 'master' into batch_commit
jackysp Dec 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ type Performance struct {
StatsLease string `toml:"stats-lease" json:"stats-lease"`
RunAutoAnalyze bool `toml:"run-auto-analyze" json:"run-auto-analyze"`
StmtCountLimit uint `toml:"stmt-count-limit" json:"stmt-count-limit"`
BatchCommit bool `toml:"batch-commit" json:"batch-commit"`
FeedbackProbability float64 `toml:"feedback-probability" json:"feedback-probability"`
QueryFeedbackLimit uint `toml:"query-feedback-limit" json:"query-feedback-limit"`
PseudoEstimateRatio float64 `toml:"pseudo-estimate-ratio" json:"pseudo-estimate-ratio"`
Expand Down Expand Up @@ -292,6 +293,7 @@ var defaultConf = Config{
StatsLease: "3s",
RunAutoAnalyze: true,
StmtCountLimit: 5000,
BatchCommit: false,
FeedbackProbability: 0.05,
QueryFeedbackLimit: 1024,
PseudoEstimateRatio: 0.8,
Expand Down
2 changes: 2 additions & 0 deletions config/config.toml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ metrics-interval = 15
max-procs = 0
# StmtCountLimit limits the max count of statement inside a transaction.
stmt-count-limit = 5000
# If true, the transaction will be committed when it reaches stmt-count-limit and starts a new transaction.
batch-commit = false
tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved

# Set keep alive option for tcp connection.
tcp-keep-alive = true
Expand Down
31 changes: 28 additions & 3 deletions session/session_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1925,13 +1925,13 @@ func (s *testSessionSuite) TestStatementErrorInTransaction(c *C) {
tk.MustQuery("select * from test where a = 1 and b = 11").Check(testkit.Rows())
}

func (s *testSessionSuite) TestStatementCountLimit(c *C) {
func (s *testSessionSuite) TestStatementCountLimitAndBatchCommit(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("create table stmt_count_limit (id int)")
saved := config.GetGlobalConfig().Performance.StmtCountLimit
savedLimit := config.GetGlobalConfig().Performance.StmtCountLimit
config.GetGlobalConfig().Performance.StmtCountLimit = 3
defer func() {
config.GetGlobalConfig().Performance.StmtCountLimit = saved
config.GetGlobalConfig().Performance.StmtCountLimit = savedLimit
}()
tk.MustExec("begin")
tk.MustExec("insert into stmt_count_limit values (1)")
Expand All @@ -1946,6 +1946,31 @@ func (s *testSessionSuite) TestStatementCountLimit(c *C) {
tk.MustExec("insert into stmt_count_limit values (3)")
_, err = tk.Exec("insert into stmt_count_limit values (4)")
c.Assert(err, NotNil)

savedBatchFlag := config.GetGlobalConfig().Performance.BatchCommit
config.GetGlobalConfig().Performance.BatchCommit = true
defer func() {
config.GetGlobalConfig().Performance.BatchCommit = savedBatchFlag
}()
tk1 := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("SET SESSION autocommit = 1")
tk.MustExec("begin")
tk.MustExec("insert into stmt_count_limit values (5)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows())
tk.MustExec("insert into stmt_count_limit values (6)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows())
tk.MustExec("insert into stmt_count_limit values (7)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7"))
tk.MustExec("insert into stmt_count_limit values (8)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7"))
tk.MustExec("insert into stmt_count_limit values (9)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7"))
tk.MustExec("insert into stmt_count_limit values (10)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7"))
tk.MustExec("commit")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7", "8", "9", "10"))
tk.MustExec("insert into stmt_count_limit values (11)")
tk1.MustQuery("select * from stmt_count_limit").Check(testkit.Rows("5", "6", "7", "8", "9", "10", "11"))
}

func (s *testSessionSuite) TestCastTimeToDate(c *C) {
Expand Down
19 changes: 14 additions & 5 deletions session/tidb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,11 +171,20 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement)
// If the user insert, insert, insert ... but never commit, TiDB would OOM.
// So we limit the statement count in a transaction here.
history := GetHistory(sctx)
if history.Count() > int(config.GetGlobalConfig().Performance.StmtCountLimit) {
err1 := se.RollbackTxn(ctx)
terror.Log(errors.Trace(err1))
return rs, errors.Errorf("statement count %d exceeds the transaction limitation, autocommit = %t",
history.Count(), sctx.GetSessionVars().IsAutocommit())
perfConfig := config.GetGlobalConfig().Performance
if history.Count() > int(perfConfig.StmtCountLimit) {
if !perfConfig.BatchCommit {
err1 := se.RollbackTxn(ctx)
terror.Log(errors.Trace(err1))
return rs, errors.Errorf("statement count %d exceeds the transaction limitation, autocommit = %t",
history.Count(), sctx.GetSessionVars().IsAutocommit())
}
err = se.NewTxn()
tiancaiamao marked this conversation as resolved.
Show resolved Hide resolved
// The transaction does not committed yet, we need to keep it in transaction.
// The last history could not be "commit" statement.
// It means it is impossible to start a new transaction at the end of the transaction (the "commit" statement).
// Because after the server executed "commit" statement, the session is out of the transaction.
se.sessionVars.SetStatusFlag(mysql.ServerStatusInTrans, true)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What will happen if the last statement is a rollback, will there be a corner case that this line put the session into a new transaction while it should be rollbacked ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rollback is the same as commit.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will update the comment and add a rollback test case.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

s1:
set @@autocommit = 0;
insert into t value (1);
rollback;

s2:
insert into t values (1);
insert into t values (2);

s1:
select  * from t;    // what it see ?

@jackysp

Copy link
Member Author

@jackysp jackysp Nov 16, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the statement count limit is 1 for this case?

}
}
return rs, errors.Trace(err)
Expand Down