log: delete lightning's redact package (#1003) (#1044)

pingcap · Jun 1, 2021 · ecfaf2e · ecfaf2e
1 parent 73e877a
commit ecfaf2e
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 125 deletions.
diff --git a/pkg/lightning/backend/kv/sql2kv.go b/pkg/lightning/backend/kv/sql2kv.go
@@ -40,6 +40,8 @@ import (
 	"github.com/pingcap/br/pkg/lightning/log"
 	"github.com/pingcap/br/pkg/lightning/metric"
 	"github.com/pingcap/br/pkg/lightning/verification"
+	"github.com/pingcap/br/pkg/logutil"
+	"github.com/pingcap/br/pkg/redact"
 )
 
 var ExtraHandleColumnInfo = model.NewExtraHandleColInfo()
@@ -217,7 +219,7 @@ func (row RowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error
 		}
 		if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error {
 			enc.AddString("kind", kindStr[kind])
-			enc.AddString("val", log.RedactString(str))
+			enc.AddString("val", redact.String(str))
 			return nil
 		})); err != nil {
 			return err
@@ -241,7 +243,7 @@ func logKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *mo
 		log.ShortError(err),
 	)
 
-	log.L().Error("failed to covert kv value", log.ZapRedactReflect("origVal", original.GetValue()),
+	log.L().Error("failed to covert kv value", logutil.RedactAny("origVal", original.GetValue()),
 		zap.Stringer("fieldType", &colInfo.FieldType), zap.String("column", colInfo.Name.O),
 		zap.Int("columnID", j+1))
 	return errors.Annotatef(

diff --git a/pkg/lightning/backend/local/local.go b/pkg/lightning/backend/local/local.go
@@ -756,9 +756,9 @@ func (local *local) WriteToTiKV(
 		return nil, nil, stats, errors.Annotate(iter.Error(), "failed to read the first key")
 	}
 	if !iter.Valid() {
-		log.L().Info("keys within region is empty, skip ingest", log.ZapRedactBinary("start", start),
-			log.ZapRedactBinary("regionStart", region.Region.StartKey), log.ZapRedactBinary("end", end),
-			log.ZapRedactBinary("regionEnd", region.Region.EndKey))
+		log.L().Info("keys within region is empty, skip ingest", logutil.Key("start", start),
+			logutil.Key("regionStart", region.Region.StartKey), logutil.Key("end", end),
+			logutil.Key("regionEnd", region.Region.EndKey))
 		return nil, nil, stats, nil
 	}
 
@@ -901,9 +901,9 @@ func (local *local) WriteToTiKV(
 		firstKey := append([]byte{}, iter.Key()...)
 		remainRange = &Range{start: firstKey, end: regionRange.end}
 		log.L().Info("write to tikv partial finish", zap.Int64("count", totalCount),
-			zap.Int64("size", size), log.ZapRedactBinary("startKey", regionRange.start), log.ZapRedactBinary("endKey", regionRange.end),
-			log.ZapRedactBinary("remainStart", remainRange.start), log.ZapRedactBinary("remainEnd", remainRange.end),
-			log.ZapRedactReflect("region", region))
+			zap.Int64("size", size), logutil.Key("startKey", regionRange.start), logutil.Key("endKey", regionRange.end),
+			logutil.Key("remainStart", remainRange.start), logutil.Key("remainEnd", remainRange.end),
+			logutil.Region(region.Region), logutil.Leader(region.Leader))
 	}
 	stats.count = totalCount
 	stats.totalBytes = size
@@ -1016,7 +1016,7 @@ func (local *local) readAndSplitIntoRange(engineFile *File) ([]Range, error) {
 
 	log.L().Info("split engine key ranges", zap.Stringer("engine", engineFile.UUID),
 		zap.Int64("totalSize", engineFileTotalSize), zap.Int64("totalCount", engineFileLength),
-		log.ZapRedactBinary("firstKey", firstKey), log.ZapRedactBinary("lastKey", lastKey),
+		logutil.Key("firstKey", firstKey), logutil.Key("lastKey", lastKey),
 		zap.Int("ranges", len(ranges)))
 
 	return ranges, nil
@@ -1134,9 +1134,9 @@ func (local *local) writeAndIngestByRange(
 	}
 	if !hasKey {
 		log.L().Info("There is no pairs in iterator",
-			log.ZapRedactBinary("start", start),
-			log.ZapRedactBinary("end", end),
-			log.ZapRedactBinary("next end", nextKey(end)))
+			logutil.Key("start", start),
+			logutil.Key("end", end),
+			logutil.Key("next end", nextKey(end)))
 		return nil
 	}
 	pairStart := append([]byte{}, iter.Key()...)
@@ -1165,7 +1165,7 @@ WriteAndIngest:
 		regions, err = paginateScanRegion(ctx, local.splitCli, startKey, endKey, 128)
 		if err != nil || len(regions) == 0 {
 			log.L().Warn("scan region failed", log.ShortError(err), zap.Int("region_len", len(regions)),
-				log.ZapRedactBinary("startKey", startKey), log.ZapRedactBinary("endKey", endKey), zap.Int("retry", retry))
+				logutil.Key("startKey", startKey), logutil.Key("endKey", endKey), zap.Int("retry", retry))
 			retry++
 			continue WriteAndIngest
 		}
@@ -1191,8 +1191,8 @@ WriteAndIngest:
 				} else {
 					retry++
 				}
-				log.L().Info("retry write and ingest kv pairs", log.ZapRedactBinary("startKey", pairStart),
-					log.ZapRedactBinary("endKey", end), log.ShortError(err), zap.Int("retry", retry))
+				log.L().Info("retry write and ingest kv pairs", logutil.Key("startKey", pairStart),
+					logutil.Key("endKey", end), log.ShortError(err), zap.Int("retry", retry))
 				continue WriteAndIngest
 			}
 			if rg != nil {
@@ -1316,8 +1316,8 @@ loopWrite:
 
 		if err != nil {
 			log.L().Warn("write and ingest region, will retry import full range", log.ShortError(err),
-				log.ZapRedactStringer("region", region.Region), log.ZapRedactBinary("start", start),
-				log.ZapRedactBinary("end", end))
+				logutil.Region(region.Region), logutil.Key("start", start),
+				logutil.Key("end", end))
 		} else {
 			metric.BytesCounter.WithLabelValues(metric.TableStateImported).Add(float64(rangeStats.totalBytes))
 		}
@@ -1653,7 +1653,7 @@ func (local *local) isIngestRetryable(
 			if newRegion != nil {
 				return newRegion, nil
 			}
-			log.L().Warn("get region by key return nil, will retry", log.ZapRedactReflect("region", region),
+			log.L().Warn("get region by key return nil, will retry", logutil.Region(region.Region), logutil.Leader(region.Leader),
 				zap.Int("retry", i))
 			select {
 			case <-ctx.Done():
@@ -2155,8 +2155,8 @@ func (sw *sstWriter) ingestInto(e *File, desc localIngestDescription) error {
 			zap.Int64("kvs", sw.totalCount),
 			zap.Uint8("description", uint8(desc)),
 			zap.Uint64("sstFileSize", meta.Size),
-			log.ZapRedactBinary("firstKey", meta.SmallestPoint.UserKey),
-			log.ZapRedactBinary("lastKey", meta.LargestPoint.UserKey))
+			logutil.Key("firstKey", meta.SmallestPoint.UserKey),
+			logutil.Key("lastKey", meta.LargestPoint.UserKey))
 
 		if err := e.db.Ingest([]string{sw.path}); err != nil {
 			return errors.Trace(err)

diff --git a/pkg/lightning/backend/local/localhelper.go b/pkg/lightning/backend/local/localhelper.go
@@ -16,7 +16,6 @@ package local
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"regexp"
 	"runtime"
 	"sort"
@@ -75,8 +74,8 @@ func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []
 	waitTime := splitRegionBaseBackOffTime
 	for i := 0; i < SplitRetryTimes; i++ {
 		log.L().Info("split and scatter region",
-			log.ZapRedactBinary("minKey", minKey),
-			log.ZapRedactBinary("maxKey", maxKey),
+			logutil.Key("minKey", minKey),
+			logutil.Key("maxKey", maxKey),
 			zap.Int("retry", i),
 		)
 		err = nil
@@ -94,18 +93,18 @@ func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []
 		var regions []*split.RegionInfo
 		regions, err = paginateScanRegion(ctx, local.splitCli, minKey, maxKey, 128)
 		if err != nil {
-			log.L().Warn("paginate scan region failed", log.ZapRedactBinary("minKey", minKey), log.ZapRedactBinary("maxKey", maxKey),
+			log.L().Warn("paginate scan region failed", logutil.Key("minKey", minKey), logutil.Key("maxKey", maxKey),
 				log.ShortError(err), zap.Int("retry", i))
 			continue
 		}
 
 		if len(regions) == 0 {
-			log.L().Warn("paginate scan region returns empty result", log.ZapRedactBinary("minKey", minKey), log.ZapRedactBinary("maxKey", maxKey),
+			log.L().Warn("paginate scan region returns empty result", logutil.Key("minKey", minKey), logutil.Key("maxKey", maxKey),
 				zap.Int("retry", i))
 			return errors.New("paginate scan region returns empty result")
 		}
 
-		log.L().Info("paginate scan region finished", log.ZapRedactBinary("minKey", minKey), log.ZapRedactBinary("maxKey", maxKey),
+		log.L().Info("paginate scan region finished", logutil.Key("minKey", minKey), logutil.Key("maxKey", maxKey),
 			zap.Int("regions", len(regions)))
 
 		if !needSplit {
@@ -124,9 +123,9 @@ func (local *local) SplitAndScatterRegionByRanges(ctx context.Context, ranges []
 			lastKeyEnc := codec.EncodeBytes([]byte{}, retryKeys[len(retryKeys)-1])
 			if bytes.Compare(firstKeyEnc, regions[0].Region.StartKey) < 0 || !beforeEnd(lastKeyEnc, regions[len(regions)-1].Region.EndKey) {
 				log.L().Warn("no valid key for split region",
-					log.ZapRedactBinary("firstKey", firstKeyEnc), log.ZapRedactBinary("lastKey", lastKeyEnc),
-					log.ZapRedactBinary("firstRegionStart", regions[0].Region.StartKey),
-					log.ZapRedactBinary("lastRegionEnd", regions[len(regions)-1].Region.EndKey))
+					logutil.Key("firstKey", firstKeyEnc), logutil.Key("lastKey", lastKeyEnc),
+					logutil.Key("firstRegionStart", regions[0].Region.StartKey),
+					logutil.Key("lastRegionEnd", regions[len(regions)-1].Region.EndKey))
 				return errors.New("check split keys failed")
 			}
 			splitKeyMap = getSplitKeys(retryKeys, regions)
@@ -283,8 +282,8 @@ func paginateScanRegion(
 ) ([]*split.RegionInfo, error) {
 	if len(endKey) != 0 && bytes.Compare(startKey, endKey) >= 0 {
 		log.L().Error("startKey > endKey when paginating scan region",
-			log.ZapRedactString("startKey", hex.EncodeToString(startKey)),
-			log.ZapRedactString("endKey", hex.EncodeToString(endKey)))
+			logutil.Key("startKey", startKey),
+			logutil.Key("endKey", endKey))
 		return nil, errors.Errorf("startKey > endKey when paginating scan region")
 	}
 
@@ -384,7 +383,7 @@ func (local *local) waitForScatterRegion(ctx context.Context, regionInfo *split.
 		ok, err := local.isScatterRegionFinished(ctx, regionID)
 		if err != nil {
 			log.L().Warn("scatter region failed: do not have the region",
-				log.ZapRedactStringer("region", regionInfo.Region))
+				logutil.Region(regionInfo.Region))
 			return
 		}
 		if ok {

diff --git a/pkg/lightning/backend/tidb/tidb.go b/pkg/lightning/backend/tidb/tidb.go
@@ -39,6 +39,7 @@ import (
 	"github.com/pingcap/br/pkg/lightning/config"
 	"github.com/pingcap/br/pkg/lightning/log"
 	"github.com/pingcap/br/pkg/lightning/verification"
+	"github.com/pingcap/br/pkg/redact"
 	"github.com/pingcap/br/pkg/version"
 )
 
@@ -59,7 +60,7 @@ type tidbRows []tidbRow
 // MarshalLogArray implements the zapcore.ArrayMarshaler interface
 func (rows tidbRows) MarshalLogArray(encoder zapcore.ArrayEncoder) error {
 	for _, r := range rows {
-		encoder.AppendString(string(r))
+		encoder.AppendString(redact.String(string(r)))
 	}
 	return nil
 }
@@ -424,8 +425,8 @@ func (be *tidbBackend) WriteRowsToDB(ctx context.Context, tableName string, colu
 	// Retry will be done externally, so we're not going to retry here.
 	_, err := be.db.ExecContext(ctx, insertStmt.String())
 	if err != nil && !common.IsContextCanceledError(err) {
-		log.L().Error("execute statement failed", log.ZapRedactString("stmt", insertStmt.String()),
-			log.ZapRedactArray("rows", rows), zap.Error(err))
+		log.L().Error("execute statement failed", zap.String("stmt", redact.String(insertStmt.String())),
+			zap.Array("rows", rows), zap.Error(err))
 	}
 	failpoint.Inject("FailIfImportedSomeRows", func() {
 		panic("forcing failure due to FailIfImportedSomeRows, before saving checkpoint")

diff --git a/pkg/lightning/lightning.go b/pkg/lightning/lightning.go
@@ -46,6 +46,7 @@ import (
 	"github.com/pingcap/br/pkg/lightning/mydump"
 	"github.com/pingcap/br/pkg/lightning/restore"
 	"github.com/pingcap/br/pkg/lightning/web"
+	"github.com/pingcap/br/pkg/redact"
 	"github.com/pingcap/br/pkg/storage"
 	"github.com/pingcap/br/pkg/utils"
 	"github.com/pingcap/br/pkg/version/build"
@@ -82,7 +83,7 @@ func New(globalCfg *config.GlobalConfig) *Lightning {
 		log.L().Fatal("failed to load TLS certificates", zap.Error(err))
 	}
 
-	log.InitRedact(globalCfg.Security.RedactInfoLog)
+	redact.InitRedact(globalCfg.Security.RedactInfoLog)
 
 	ctx, shutdown := context.WithCancel(context.Background())
 	return &Lightning{

diff --git a/pkg/lightning/log/redact.go b/pkg/lightning/log/redact.go