Merge remote-tracking branch 'pingcap/master' into issue-31629

Makefile

@@ -14,7 +14,7 @@
 
 include Makefile.common
 
-.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling
+.PHONY: all clean test gotest server dev benchkv benchraw check checklist parser tidy ddltest build_br build_lightning build_lightning-ctl build_dumpling ut
 
 default: server buildsucc
 
@@ -28,7 +28,7 @@ all: dev server benchkv
 parser:
 	@echo "remove this command later, when our CI script doesn't call it"
 
-dev: checklist check explaintest devgotest gogenerate br_unit_test test_part_parser_dev
+dev: checklist check explaintest gogenerate br_unit_test test_part_parser_dev
 	@>&2 echo "Great, all tests passed."
 
 # Install the check tools.
@@ -115,34 +115,26 @@ explaintest: server_check
 ddltest:
 	@cd cmd/ddltest && $(GO) test -o ../../bin/ddltest -c
 
-devgotest: failpoint-enable
-# grep regex: Filter out all tidb logs starting with:
-# - '[20' (like [2021/09/15 ...] [INFO]..)
-# - 'PASS:' to ignore passed tests
-# - 'ok ' to ignore passed directories
-	@echo "Running in native mode."
-	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -check.p true > gotest.log || { $(FAILPOINT_DISABLE); grep -v '^\([[]20\|PASS:\|ok \)' 'gotest.log'; exit 1; }
-	@$(FAILPOINT_DISABLE)
+CLEAN_UT_BINARY := find . -name '*.test.bin'| xargs rm
 
-ut: failpoint-enable tools/bin/ut
-	tools/bin/ut $(X);
+ut: tools/bin/ut tools/bin/xprog failpoint-enable
+	tools/bin/ut $(X) || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
+	@$(CLEAN_UT_BINARY)
 
 gotest: failpoint-enable
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
 	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
-gotest_in_verify_ci: failpoint-enable tools/bin/gotestsum
+gotest_in_verify_ci: tools/bin/xprog tools/bin/ut failpoint-enable
 	@echo "Running gotest_in_verify_ci"
 	@mkdir -p $(TEST_COVERAGE_DIR)
 	@export TZ='Asia/Shanghai'; \
-	CGO_ENABLED=1 tools/bin/gotestsum --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" -- -v -p $(P) \
-	-ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -coverprofile="$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" \
-	$(PACKAGES_TIDB_TESTS) -check.p true || { $(FAILPOINT_DISABLE); exit 1; }
+	tools/bin/ut --junitfile "$(TEST_COVERAGE_DIR)/tidb-junit-report.xml" --coverprofile "$(TEST_COVERAGE_DIR)/tidb_cov.unit_test.out" || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
+	@$(CLEAN_UT_BINARY)
 
 race: failpoint-enable
 	@export log_level=debug; \
@@ -216,6 +208,10 @@ tools/bin/ut: tools/check/ut.go
 	cd tools/check; \
 	$(GO) build -o ../bin/ut ut.go
 
+tools/bin/xprog: tools/check/xprog.go
+	cd tools/check; \
+	$(GO) build -o ../bin/xprog xprog.go
+
 tools/bin/megacheck: tools/check/go.mod
 	cd tools/check; \
 	$(GO) build -o ../bin/megacheck honnef.co/go/tools/cmd/megacheck

bindinfo/handle.go

@@ -131,15 +131,11 @@ func (h *BindHandle) Update(fullLoad bool) (err error) {
 	}
 
 	exec := h.sctx.Context.(sqlexec.RestrictedSQLExecutor)
-	stmt, err := exec.ParseWithParams(context.TODO(), true, `SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source
-	FROM mysql.bind_info WHERE update_time > %? ORDER BY update_time, create_time`, updateTime)
-	if err != nil {
-		return err
-	}
 
-	// No need to acquire the session context lock for ExecRestrictedStmt, it
+	// No need to acquire the session context lock for ExecRestrictedSQL, it
 	// uses another background session.
-	rows, _, err := exec.ExecRestrictedStmt(context.Background(), stmt)
+	rows, _, err := exec.ExecRestrictedSQL(context.TODO(), nil, `SELECT original_sql, bind_sql, default_db, status, create_time, update_time, charset, collation, source
+	FROM mysql.bind_info WHERE update_time > %? ORDER BY update_time, create_time`, updateTime)
 
 	if err != nil {
 		h.bindInfo.Unlock()
@@ -700,14 +696,9 @@ func (h *BindHandle) extractCaptureFilterFromStorage() (filter *captureFilter) {
 		tables:    make(map[stmtctx.TableEntry]struct{}),
 	}
 	exec := h.sctx.Context.(sqlexec.RestrictedSQLExecutor)
-	stmt, err := exec.ParseWithParams(context.TODO(), true, `SELECT filter_type, filter_value FROM mysql.capture_plan_baselines_blacklist order by filter_type`)
-	if err != nil {
-		logutil.BgLogger().Warn("[sql-bind] failed to parse query for mysql.capture_plan_baselines_blacklist load", zap.Error(err))
-		return
-	}
-	// No need to acquire the session context lock for ExecRestrictedStmt, it
+	// No need to acquire the session context lock for ExecRestrictedSQL, it
 	// uses another background session.
-	rows, _, err := exec.ExecRestrictedStmt(context.TODO(), stmt)
+	rows, _, err := exec.ExecRestrictedSQL(context.TODO(), nil, `SELECT filter_type, filter_value FROM mysql.capture_plan_baselines_blacklist order by filter_type`)
 	if err != nil {
 		logutil.BgLogger().Warn("[sql-bind] failed to load mysql.capture_plan_baselines_blacklist", zap.Error(err))
 		return
@@ -926,9 +917,9 @@ func (h *BindHandle) SaveEvolveTasksToStore() {
 }
 
 func getEvolveParameters(ctx sessionctx.Context) (time.Duration, time.Time, time.Time, error) {
-	stmt, err := ctx.(sqlexec.RestrictedSQLExecutor).ParseWithParams(
+	rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(
 		context.TODO(),
-		true,
+		nil,
 		"SELECT variable_name, variable_value FROM mysql.global_variables WHERE variable_name IN (%?, %?, %?)",
 		variable.TiDBEvolvePlanTaskMaxTime,
 		variable.TiDBEvolvePlanTaskStartTime,
@@ -937,10 +928,6 @@ func getEvolveParameters(ctx sessionctx.Context) (time.Duration, time.Time, time
 	if err != nil {
 		return 0, time.Time{}, time.Time{}, err
 	}
-	rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedStmt(context.TODO(), stmt)
-	if err != nil {
-		return 0, time.Time{}, time.Time{}, err
-	}
 	maxTime, startTimeStr, endTimeStr := int64(variable.DefTiDBEvolvePlanTaskMaxTime), variable.DefTiDBEvolvePlanTaskStartTime, variable.DefAutoAnalyzeEndTime
 	for _, row := range rows {
 		switch row.GetString(0) {

bindinfo/handle_test.go

@@ -38,7 +38,7 @@ func utilCleanBindingEnv(tk *testkit.TestKit, dom *domain.Domain) {
 func utilNormalizeWithDefaultDB(t *testing.T, sql, db string) (string, string) {
 	testParser := parser.New()
 	stmt, err := testParser.ParseOneStmt(sql, "", "")
-	require.Nil(t, err)
+	require.NoError(t, err)
 	normalized, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", ""))
 	return normalized, digest.String()
 }
@@ -82,7 +82,7 @@ func TestBindingLastUpdateTime(t *testing.T) {
 
 	bindHandle := bindinfo.NewBindHandle(tk.Session())
 	err := bindHandle.Update(true)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	sql, hash := parser.NormalizeDigest("select * from test . t0")
 	bindData := bindHandle.GetBindRecord(hash.String(), sql, "test")
 	require.Equal(t, 1, len(bindData.Bindings))
@@ -121,7 +121,7 @@ func TestBindingLastUpdateTimeWithInvalidBind(t *testing.T) {
 	require.Equal(t, updateTime1, "2000-01-01 09:00:00.000")
 
 	rows2 := tk.MustQuery("show global bindings").Rows()
-	require.Equal(t, len(rows2), 0)
+	require.Len(t, rows2, 0)
 }
 
 func TestBindParse(t *testing.T) {
@@ -146,7 +146,7 @@ func TestBindParse(t *testing.T) {
 	tk.MustExec(sql)
 	bindHandle := bindinfo.NewBindHandle(tk.Session())
 	err := bindHandle.Update(true)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 1, bindHandle.Size())
 
 	sql, hash := parser.NormalizeDigest("select * from test . t")
@@ -162,7 +162,7 @@ func TestBindParse(t *testing.T) {
 	require.NotNil(t, bind.CreateTime)
 	require.NotNil(t, bind.UpdateTime)
 	dur, err := bind.SinceUpdateTime()
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.GreaterOrEqual(t, int64(dur), int64(0))
 
 	// Test fields with quotes or slashes.
@@ -387,19 +387,19 @@ func TestGlobalBinding(t *testing.T) {
 		metrics.BindMemoryUsage.Reset()
 
 		_, err := tk.Exec("create global " + testSQL.createSQL)
-		require.Nil(t, err, "err %v", err)
+		require.NoError(t, err, "err %v", err)
 
 		if testSQL.overlaySQL != "" {
 			_, err = tk.Exec("create global " + testSQL.overlaySQL)
-			require.Nil(t, err)
+			require.NoError(t, err)
 		}
 
 		pb := &dto.Metric{}
 		err = metrics.BindTotalGauge.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, float64(1), pb.GetGauge().GetValue())
 		err = metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, testSQL.memoryUsage, pb.GetGauge().GetValue())
 
 		sql, hash := utilNormalizeWithDefaultDB(t, testSQL.querySQL, "test")
@@ -417,10 +417,10 @@ func TestGlobalBinding(t *testing.T) {
 		require.NotNil(t, bind.UpdateTime)
 
 		rs, err := tk.Exec("show global bindings")
-		require.Nil(t, err)
+		require.NoError(t, err)
 		chk := rs.NewChunk(nil)
 		err = rs.Next(context.TODO(), chk)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, 1, chk.NumRows())
 		row := chk.GetRow(0)
 		require.Equal(t, testSQL.originSQL, row.GetString(0))
@@ -434,7 +434,7 @@ func TestGlobalBinding(t *testing.T) {
 
 		bindHandle := bindinfo.NewBindHandle(tk.Session())
 		err = bindHandle.Update(true)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, 1, bindHandle.Size())
 
 		bindData = bindHandle.GetBindRecord(hash, sql, "test")
@@ -450,35 +450,35 @@ func TestGlobalBinding(t *testing.T) {
 		require.NotNil(t, bind.UpdateTime)
 
 		_, err = tk.Exec("drop global " + testSQL.dropSQL)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		bindData = dom.BindHandle().GetBindRecord(hash, sql, "test")
 		require.Nil(t, bindData)
 
 		err = metrics.BindTotalGauge.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, float64(0), pb.GetGauge().GetValue())
 		err = metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		// From newly created global bind handle.
 		require.Equal(t, testSQL.memoryUsage, pb.GetGauge().GetValue())
 
 		bindHandle = bindinfo.NewBindHandle(tk.Session())
 		err = bindHandle.Update(true)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, 0, bindHandle.Size())
 
 		bindData = bindHandle.GetBindRecord(hash, sql, "test")
 		require.Nil(t, bindData)
 
 		rs, err = tk.Exec("show global bindings")
-		require.Nil(t, err)
+		require.NoError(t, err)
 		chk = rs.NewChunk(nil)
 		err = rs.Next(context.TODO(), chk)
-		require.Nil(t, err)
+		require.NoError(t, err)
 		require.Equal(t, 0, chk.NumRows())
 
 		_, err = tk.Exec("delete from mysql.bind_info where source != 'builtin'")
-		require.Nil(t, err)
+		require.NoError(t, err)
 	}
 }
 

bindinfo/session_handle_test.go

@@ -106,7 +106,7 @@ func TestSessionBinding(t *testing.T) {
 		metrics.BindMemoryUsage.Reset()
 
 		_, err := tk.Exec("create session " + testSQL.createSQL)
-		require.Nil(t, err, "err %v", err)
+		require.NoError(t, err, "err %v", err)
 
 		if testSQL.overlaySQL != "" {
 			_, err = tk.Exec("create session " + testSQL.overlaySQL)

br/cmd/tidb-lightning/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/config"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
+	"github.com/pingcap/tidb/br/pkg/lightning/web"
 	"go.uber.org/zap"
 )
 
@@ -78,6 +79,9 @@ func main() {
 		fmt.Fprintln(os.Stderr, "failed to start HTTP server:", err)
 		return
 	}
+	if len(globalCfg.App.StatusAddr) > 0 {
+		web.EnableCurrentProgress()
+	}
 
 	err = func() error {
 		if globalCfg.App.ServerMode {