diff --git a/cmd/debug.go b/cmd/debug.go index 226a53bed..8ab5b7f87 100644 --- a/cmd/debug.go +++ b/cmd/debug.go @@ -22,6 +22,7 @@ import ( "go.uber.org/zap" berrors "github.com/pingcap/br/pkg/errors" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/restore" "github.com/pingcap/br/pkg/rtree" "github.com/pingcap/br/pkg/task" @@ -106,8 +107,8 @@ func newCheckSumCommand() *cobra.Command { zap.Uint64("totalBytes", file.GetTotalBytes()), zap.Uint64("startVersion", file.GetStartVersion()), zap.Uint64("endVersion", file.GetEndVersion()), - zap.Stringer("startKey", utils.WrapKey(file.GetStartKey())), - zap.Stringer("endKey", utils.WrapKey(file.GetEndKey())), + zap.Stringer("startKey", logutil.WrapKey(file.GetStartKey())), + zap.Stringer("endKey", logutil.WrapKey(file.GetEndKey())), ) var data []byte @@ -186,7 +187,7 @@ func newBackupMetaCommand() *cobra.Command { log.Error( "file ranges overlapped", zap.Stringer("out", out), - utils.ZapFile(file), + logutil.File(file), ) } } diff --git a/docker-compose.yaml b/docker-compose.yaml index ab6360d6d..1b16aaed0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,13 +4,15 @@ version: '3.2' services: control: - image: control:latest + image: control:nightly build: context: . dockerfile: ./docker/Dockerfile volumes: - - ./docker/data:/data - - ./docker/logs:/tmp + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/tmp + - /tmp/br/docker/logs:/logs + - ./bin:/go/src/github.com/pingcap/br/bin command: -c "/usr/bin/tail -f /dev/null" depends_on: - "tidb" @@ -19,13 +21,13 @@ services: - ./docker/minio.env pd0: - image: pingcap/pd:latest + image: pingcap/pd:nightly ports: - "2379" volumes: - ./docker/config/pd.toml:/pd.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --name=pd0 - --client-urls=http://0.0.0.0:2379 @@ -45,11 +47,11 @@ services: restart: on-failure tikv0: - image: pingcap/tikv:latest + image: pingcap/tikv:nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=tikv0:20160 @@ -70,11 +72,11 @@ services: - ./docker/minio.env tikv1: - image: pingcap/tikv:latest + image: pingcap/tikv:nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=tikv1:20160 @@ -95,11 +97,11 @@ services: - ./docker/minio.env tikv2: - image: pingcap/tikv:latest + image: pingcap/tikv:nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=tikv2:20160 @@ -120,11 +122,11 @@ services: - ./docker/minio.env tikv3: - image: pingcap/tikv:latest + image: pingcap/tikv:nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=tikv3:20160 @@ -145,11 +147,11 @@ services: - ./docker/minio.env tikv4: - image: pingcap/tikv:latest + image: pingcap/tikv:nightly volumes: - ./docker/config/tikv.toml:/tikv.toml:ro - - ./docker/data:/data - - ./docker/logs:/logs + - /tmp/br/docker/data:/data + - /tmp/br/docker/logs:/logs command: - --addr=0.0.0.0:20160 - --advertise-addr=tikv4:20160 @@ -170,13 +172,13 @@ services: - ./docker/minio.env tidb: - image: pingcap/tidb:latest + image: pingcap/tidb:nightly ports: - - "4000:4000" - - "10080:10080" + - "4000" + - "10080" volumes: - ./docker/config/tidb.toml:/tidb.toml:ro - - ./docker/logs:/logs + - /tmp/br/docker/logs:/logs command: - --store=tikv - --path=pd0:2379 @@ -202,15 +204,7 @@ services: ports: - 24927:24927 volumes: - - ./docker/data/s3:/data/s3 + - /tmp/br/docker/data/s3:/data/s3 command: server --address=:24927 /data/s3 env_file: - ./docker/minio.env - - tidb-vision: - image: pingcap/tidb-vision:latest - environment: - PD_ENDPOINT: pd0:2379 - ports: - - "8010:8010" - restart: on-failure diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 47bb71c61..5eac828d0 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/br/pkg/conn" berrors "github.com/pingcap/br/pkg/errors" "github.com/pingcap/br/pkg/glue" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/rtree" "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" @@ -482,8 +483,8 @@ func (bc *Client) BackupRange( } }() log.Info("backup started", - zap.Stringer("StartKey", utils.WrapKey(startKey)), - zap.Stringer("EndKey", utils.WrapKey(endKey)), + zap.Stringer("StartKey", logutil.WrapKey(startKey)), + zap.Stringer("EndKey", logutil.WrapKey(endKey)), zap.Uint64("RateLimit", req.RateLimit), zap.Uint32("Concurrency", req.Concurrency)) @@ -518,8 +519,8 @@ func (bc *Client) BackupRange( if req.IsRawKv { log.Info("backup raw ranges", - zap.Stringer("startKey", utils.WrapKey(startKey)), - zap.Stringer("endKey", utils.WrapKey(endKey)), + zap.Stringer("startKey", logutil.WrapKey(startKey)), + zap.Stringer("endKey", logutil.WrapKey(endKey)), zap.String("cf", req.Cf)) } else { log.Info("backup time range", @@ -554,14 +555,14 @@ func (bc *Client) findRegionLeader(ctx context.Context, key []byte) (*metapb.Pee } if region.Leader != nil { log.Info("find leader", - zap.Reflect("Leader", region.Leader), zap.Stringer("Key", utils.WrapKey(key))) + zap.Reflect("Leader", region.Leader), zap.Stringer("Key", logutil.WrapKey(key))) return region.Leader, nil } - log.Warn("no region found", zap.Stringer("Key", utils.WrapKey(key))) + log.Warn("no region found", zap.Stringer("Key", logutil.WrapKey(key))) time.Sleep(time.Millisecond * time.Duration(100*i)) continue } - return nil, errors.Annotatef(berrors.ErrBackupNoLeader, "can not find leader for key %s", utils.WrapKey(key)) + return nil, errors.Annotatef(berrors.ErrBackupNoLeader, "can not find leader for key %s", logutil.WrapKey(key)) } func (bc *Client) fineGrainedBackup( @@ -644,8 +645,8 @@ func (bc *Client) fineGrainedBackup( zap.Reflect("error", resp.Error)) } log.Info("put fine grained range", - zap.Stringer("StartKey", utils.WrapKey(resp.StartKey)), - zap.Stringer("EndKey", utils.WrapKey(resp.EndKey)), + zap.Stringer("StartKey", logutil.WrapKey(resp.StartKey)), + zap.Stringer("EndKey", logutil.WrapKey(resp.EndKey)), ) rangeTree.Put(resp.StartKey, resp.EndKey, resp.Files) @@ -806,8 +807,8 @@ func SendBackup( backupLoop: for retry := 0; retry < backupRetryTimes; retry++ { log.Info("try backup", - zap.Stringer("StartKey", utils.WrapKey(req.StartKey)), - zap.Stringer("EndKey", utils.WrapKey(req.EndKey)), + zap.Stringer("StartKey", logutil.WrapKey(req.StartKey)), + zap.Stringer("EndKey", logutil.WrapKey(req.EndKey)), zap.Uint64("storeID", storeID), zap.Int("retry time", retry), ) @@ -856,8 +857,8 @@ backupLoop: } // TODO: handle errors in the resp. log.Info("range backuped", - zap.Stringer("StartKey", utils.WrapKey(resp.GetStartKey())), - zap.Stringer("EndKey", utils.WrapKey(resp.GetEndKey()))) + zap.Stringer("StartKey", logutil.WrapKey(resp.GetStartKey())), + zap.Stringer("EndKey", logutil.WrapKey(resp.GetEndKey()))) err = respFn(resp) if err != nil { return err diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index 02e3ac580..1abba92cf 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -24,6 +24,7 @@ import ( berrors "github.com/pingcap/br/pkg/errors" "github.com/pingcap/br/pkg/glue" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/pdutil" "github.com/pingcap/br/pkg/utils" ) @@ -237,7 +238,7 @@ func (mgr *Mgr) ResetBackupClient(ctx context.Context, storeID uint64) (backup.B conn, err = mgr.getGrpcConnLocked(ctx, storeID) if err != nil { log.Warn("failed to reset grpc connection, retry it", - zap.Int("retry time", retry), zap.Error(err)) + zap.Int("retry time", retry), logutil.ShortError(err)) time.Sleep(time.Duration(retry+3) * time.Second) continue } diff --git a/pkg/utils/logging.go b/pkg/logutil/logging.go similarity index 84% rename from pkg/utils/logging.go rename to pkg/logutil/logging.go index 18491c63a..70f475123 100644 --- a/pkg/utils/logging.go +++ b/pkg/logutil/logging.go @@ -1,6 +1,6 @@ // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. -package utils +package logutil import ( "encoding/hex" @@ -110,27 +110,32 @@ func WrapKeys(keys [][]byte) zapcore.ArrayMarshaler { return zapArrayMarshalKeysMixIn(keys) } -// ZapRewriteRule make the zap fields for a rewrite rule. -func ZapRewriteRule(rewriteRule *import_sstpb.RewriteRule) zapcore.Field { +// RewriteRule make the zap fields for a rewrite rule. +func RewriteRule(rewriteRule *import_sstpb.RewriteRule) zapcore.Field { return zap.Object("rewriteRule", zapMarshalRewriteRuleMixIn{rewriteRule}) } -// ZapRegion make the zap fields for a region. -func ZapRegion(region *metapb.Region) zapcore.Field { +// Region make the zap fields for a region. +func Region(region *metapb.Region) zapcore.Field { return zap.Object("region", zapMarshalRegionMixIn{region}) } -// ZapFile make the zap fields for a file. -func ZapFile(file *backup.File) zapcore.Field { +// File make the zap fields for a file. +func File(file *backup.File) zapcore.Field { return zap.Object("file", zapMarshalFileMixIn{file}) } -// ZapSSTMeta make the zap fields for a SST meta. -func ZapSSTMeta(sstMeta *import_sstpb.SSTMeta) zapcore.Field { +// SSTMeta make the zap fields for a SST meta. +func SSTMeta(sstMeta *import_sstpb.SSTMeta) zapcore.Field { return zap.Object("sstMeta", zapMarshalSSTMetaMixIn{sstMeta}) } -// ZapFiles make the zap field for a set of file. -func ZapFiles(fs []*backup.File) zapcore.Field { +// Files make the zap field for a set of file. +func Files(fs []*backup.File) zapcore.Field { return zap.Object("fs", files(fs)) } + +// ShortError make the zap field to display error without verbose representation (e.g. the stack trace). +func ShortError(err error) zapcore.Field { + return zap.String("error", err.Error()) +} diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 39d22de5c..707abf3ab 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -37,6 +37,7 @@ import ( "github.com/pingcap/br/pkg/conn" berrors "github.com/pingcap/br/pkg/errors" "github.com/pingcap/br/pkg/glue" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" @@ -517,7 +518,7 @@ func (rc *Client) RestoreFiles( if err == nil { log.Info("Restore files", zap.Duration("take", elapsed), - utils.ZapFiles(files)) + logutil.Files(files)) summary.CollectSuccessUnit("files", len(files), elapsed) } }() @@ -537,7 +538,7 @@ func (rc *Client) RestoreFiles( func() error { fileStart := time.Now() defer func() { - log.Info("import file done", utils.ZapFile(fileReplica), + log.Info("import file done", logutil.File(fileReplica), zap.Duration("take", time.Since(fileStart))) updateCh.Inc() }() diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 497ec9775..a3f4248cc 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -24,6 +24,7 @@ import ( "google.golang.org/grpc/keepalive" berrors "github.com/pingcap/br/pkg/errors" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" ) @@ -200,7 +201,7 @@ func (importer *FileImporter) Import( file *backup.File, rewriteRules *RewriteRules, ) error { - log.Debug("import file", utils.ZapFile(file)) + log.Debug("import file", logutil.File(file)) // Rewrite the start key and end key of file to scan regions var startKey, endKey []byte var err error @@ -218,9 +219,9 @@ func (importer *FileImporter) Import( return err } log.Debug("rewrite file keys", - utils.ZapFile(file), - zap.Stringer("startKey", utils.WrapKey(startKey)), - zap.Stringer("endKey", utils.WrapKey(endKey))) + logutil.File(file), + zap.Stringer("startKey", logutil.WrapKey(startKey)), + zap.Stringer("endKey", logutil.WrapKey(endKey))) err = utils.WithRetry(ctx, func() error { tctx, cancel := context.WithTimeout(ctx, importScanRegionTime) @@ -232,7 +233,7 @@ func (importer *FileImporter) Import( return errors.Trace(errScanRegion) } - log.Debug("scan regions", utils.ZapFile(file), zap.Int("count", len(regionInfos))) + log.Debug("scan regions", logutil.File(file), zap.Int("count", len(regionInfos))) // Try to download and ingest the file in every region regionLoop: for _, regionInfo := range regionInfos { @@ -254,20 +255,20 @@ func (importer *FileImporter) Import( case berrors.ErrKVRewriteRuleNotFound, berrors.ErrKVRangeIsEmpty: // Skip this region log.Warn("download file skipped", - utils.ZapFile(file), - utils.ZapRegion(info.Region), - zap.Stringer("startKey", utils.WrapKey(startKey)), - zap.Stringer("endKey", utils.WrapKey(endKey)), - zap.Error(e)) + logutil.File(file), + logutil.Region(info.Region), + zap.Stringer("startKey", logutil.WrapKey(startKey)), + zap.Stringer("endKey", logutil.WrapKey(endKey)), + logutil.ShortError(e)) continue regionLoop } } log.Error("download file failed", - utils.ZapFile(file), - utils.ZapRegion(info.Region), - zap.Stringer("startKey", utils.WrapKey(startKey)), - zap.Stringer("endKey", utils.WrapKey(endKey)), - zap.Error(errDownload)) + logutil.File(file), + logutil.Region(info.Region), + zap.Stringer("startKey", logutil.WrapKey(startKey)), + zap.Stringer("endKey", logutil.WrapKey(endKey)), + logutil.ShortError(errDownload)) return errDownload } @@ -303,7 +304,7 @@ func (importer *FileImporter) Import( } } log.Debug("ingest sst returns not leader error, retry it", - utils.ZapRegion(info.Region), + logutil.Region(info.Region), zap.Stringer("newLeader", newInfo.Leader)) if !checkRegionEpoch(newInfo, info) { @@ -329,9 +330,9 @@ func (importer *FileImporter) Import( if errIngest != nil { log.Error("ingest file failed", - utils.ZapFile(file), + logutil.File(file), zap.Stringer("range", downloadMeta.GetRange()), - utils.ZapRegion(info.Region), + logutil.Region(info.Region), zap.Error(errIngest)) return errIngest } @@ -381,9 +382,9 @@ func (importer *FileImporter) downloadSST( RewriteRule: rule, } log.Debug("download SST", - utils.ZapSSTMeta(&sstMeta), - utils.ZapFile(file), - utils.ZapRegion(regionInfo.Region), + logutil.SSTMeta(&sstMeta), + logutil.File(file), + logutil.Region(regionInfo.Region), ) var resp *import_sstpb.DownloadResponse for _, peer := range regionInfo.Region.GetPeers() { @@ -434,7 +435,7 @@ func (importer *FileImporter) downloadRawKVSST( RewriteRule: rule, IsRawKv: true, } - log.Debug("download SST", utils.ZapSSTMeta(&sstMeta), utils.ZapRegion(regionInfo.Region)) + log.Debug("download SST", logutil.SSTMeta(&sstMeta), logutil.Region(regionInfo.Region)) var err error var resp *import_sstpb.DownloadResponse for _, peer := range regionInfo.Region.GetPeers() { @@ -472,7 +473,7 @@ func (importer *FileImporter) ingestSST( Context: reqCtx, Sst: sstMeta, } - log.Debug("ingest SST", utils.ZapSSTMeta(sstMeta), zap.Reflect("leader", leader)) + log.Debug("ingest SST", logutil.SSTMeta(sstMeta), zap.Reflect("leader", leader)) resp, err := importer.importClient.IngestSST(ctx, leader.GetStoreId(), req) if err != nil { return nil, errors.Trace(err) diff --git a/pkg/restore/range.go b/pkg/restore/range.go index 00b29000f..7f579c0ac 100644 --- a/pkg/restore/range.go +++ b/pkg/restore/range.go @@ -13,8 +13,8 @@ import ( "go.uber.org/zap" berrors "github.com/pingcap/br/pkg/errors" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/rtree" - "github.com/pingcap/br/pkg/utils" ) // SortRanges checks if the range overlapped and sort them. @@ -28,26 +28,26 @@ func SortRanges(ranges []rtree.Range, rewriteRules *RewriteRules) ([]rtree.Range if startID == endID { rg.StartKey, rule = replacePrefix(rg.StartKey, rewriteRules) if rule == nil { - log.Warn("cannot find rewrite rule", zap.Stringer("key", utils.WrapKey(rg.StartKey))) + log.Warn("cannot find rewrite rule", zap.Stringer("key", logutil.WrapKey(rg.StartKey))) } else { log.Debug( "rewrite start key", - zap.Stringer("key", utils.WrapKey(rg.StartKey)), - utils.ZapRewriteRule(rule)) + zap.Stringer("key", logutil.WrapKey(rg.StartKey)), + logutil.RewriteRule(rule)) } rg.EndKey, rule = replacePrefix(rg.EndKey, rewriteRules) if rule == nil { - log.Warn("cannot find rewrite rule", zap.Stringer("key", utils.WrapKey(rg.EndKey))) + log.Warn("cannot find rewrite rule", zap.Stringer("key", logutil.WrapKey(rg.EndKey))) } else { log.Debug( "rewrite end key", - zap.Stringer("key", utils.WrapKey(rg.EndKey)), - utils.ZapRewriteRule(rule)) + zap.Stringer("key", logutil.WrapKey(rg.EndKey)), + logutil.RewriteRule(rule)) } } else { log.Warn("table id does not match", - zap.Stringer("startKey", utils.WrapKey(rg.StartKey)), - zap.Stringer("endKey", utils.WrapKey(rg.EndKey)), + zap.Stringer("startKey", logutil.WrapKey(rg.StartKey)), + zap.Stringer("endKey", logutil.WrapKey(rg.EndKey)), zap.Int64("startID", startID), zap.Int64("endID", endID)) return nil, errors.Annotate(berrors.ErrRestoreTableIDMismatch, "table id mismatch") diff --git a/pkg/restore/split.go b/pkg/restore/split.go index e0ff17357..c1bb87ebf 100644 --- a/pkg/restore/split.go +++ b/pkg/restore/split.go @@ -18,8 +18,8 @@ import ( "go.uber.org/zap" berrors "github.com/pingcap/br/pkg/errors" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/rtree" - "github.com/pingcap/br/pkg/utils" ) // Constants for split retry machinery. @@ -120,9 +120,9 @@ SplitRegions: if strings.Contains(errSplit.Error(), "no valid key") { for _, key := range keys { log.Error("no valid key", - zap.Stringer("startKey", utils.WrapKey(region.Region.StartKey)), - zap.Stringer("endKey", utils.WrapKey(region.Region.EndKey)), - zap.Stringer("key", utils.WrapKey(codec.EncodeBytes([]byte{}, key)))) + zap.Stringer("startKey", logutil.WrapKey(region.Region.StartKey)), + zap.Stringer("endKey", logutil.WrapKey(region.Region.EndKey)), + zap.Stringer("key", logutil.WrapKey(codec.EncodeBytes([]byte{}, key)))) } return errors.Trace(errSplit) } @@ -132,11 +132,11 @@ SplitRegions: } time.Sleep(interval) if i > 3 { - log.Warn("splitting regions failed, retry it", zap.Error(errSplit), zap.Array("keys", utils.WrapKeys(keys))) + log.Warn("splitting regions failed, retry it", zap.Error(errSplit), zap.Array("keys", logutil.WrapKeys(keys))) } continue SplitRegions } - log.Debug("split regions", utils.ZapRegion(region.Region), zap.Array("keys", utils.WrapKeys(keys))) + log.Debug("split regions", logutil.Region(region.Region), zap.Array("keys", logutil.WrapKeys(keys))) scatterRegions = append(scatterRegions, newRegions...) onSplit(keys) } @@ -229,7 +229,7 @@ func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo * ok, err := rs.isScatterRegionFinished(ctx1, regionID) if err != nil { log.Warn("scatter region failed: do not have the region", - utils.ZapRegion(regionInfo.Region)) + logutil.Region(regionInfo.Region)) return } if ok { @@ -254,7 +254,7 @@ func (rs *RegionSplitter) splitAndScatterRegions( // Wait for a while until the regions successfully split. rs.waitForSplit(ctx, region.Region.Id) if err = rs.client.ScatterRegion(ctx, region); err != nil { - log.Warn("scatter region failed", utils.ZapRegion(region.Region), zap.Error(err)) + log.Warn("scatter region failed", logutil.Region(region.Region), zap.Error(err)) } } return newRegions, nil @@ -282,9 +282,9 @@ func getSplitKeys(rewriteRules *RewriteRules, ranges []rtree.Range, regions []*R } splitKeyMap[region.Region.GetId()] = append(splitKeys, key) log.Debug("get key for split region", - zap.Stringer("key", utils.WrapKey(key)), - zap.Stringer("startKey", utils.WrapKey(region.Region.StartKey)), - zap.Stringer("endKey", utils.WrapKey(region.Region.EndKey))) + zap.Stringer("key", logutil.WrapKey(key)), + zap.Stringer("startKey", logutil.WrapKey(region.Region.StartKey)), + zap.Stringer("endKey", logutil.WrapKey(region.Region.EndKey))) } } return splitKeyMap diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 0b8fcafad..1de849f71 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -25,6 +25,7 @@ import ( berrors "github.com/pingcap/br/pkg/errors" "github.com/pingcap/br/pkg/glue" + "github.com/pingcap/br/pkg/logutil" "github.com/pingcap/br/pkg/rtree" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" @@ -123,15 +124,15 @@ func GetSSTMetaFromFile( if bytes.Compare(rangeStart, rangeEnd) > 0 { log.Panic("range start exceed range end", - utils.ZapFile(file), - zap.Stringer("rangeStart", utils.WrapKey(rangeStart)), - zap.Stringer("rangeEnd", utils.WrapKey(rangeEnd))) + logutil.File(file), + zap.Stringer("rangeStart", logutil.WrapKey(rangeStart)), + zap.Stringer("rangeEnd", logutil.WrapKey(rangeEnd))) } log.Debug("get sstMeta", - utils.ZapFile(file), - zap.Stringer("rangeStart", utils.WrapKey(rangeStart)), - zap.Stringer("rangeEnd", utils.WrapKey(rangeEnd))) + logutil.File(file), + zap.Stringer("rangeStart", logutil.WrapKey(rangeStart)), + zap.Stringer("rangeEnd", logutil.WrapKey(rangeEnd))) return import_sstpb.SSTMeta{ Uuid: id, @@ -202,14 +203,14 @@ func MapTableToFiles(files []*backup.File) map[int64][]*backup.File { if tableID != tableEndID { log.Panic("key range spread between many files.", zap.String("file name", file.Name), - zap.Stringer("start key", utils.WrapKey(file.GetStartKey())), - zap.Stringer("end key", utils.WrapKey(file.GetEndKey()))) + zap.Stringer("start key", logutil.WrapKey(file.GetStartKey())), + zap.Stringer("end key", logutil.WrapKey(file.GetEndKey()))) } if tableID == 0 { log.Panic("invalid table key of file", zap.String("file name", file.Name), - zap.Stringer("start key", utils.WrapKey(file.GetStartKey())), - zap.Stringer("end key", utils.WrapKey(file.GetEndKey()))) + zap.Stringer("start key", logutil.WrapKey(file.GetStartKey())), + zap.Stringer("end key", logutil.WrapKey(file.GetEndKey()))) } result[tableID] = append(result[tableID], file) } @@ -281,7 +282,7 @@ func validateAndGetFileRange(file *backup.File, rules *RewriteRules) (rtree.Rang log.Error("table ids mismatch", zap.Int64("startID", startID), zap.Int64("endID", endID), - utils.ZapFile(file)) + logutil.File(file)) return rtree.Range{}, errors.Annotate(berrors.ErrRestoreTableIDMismatch, "validateAndGetFileRange") } r := rtree.Range{StartKey: file.GetStartKey(), EndKey: file.GetEndKey()} @@ -306,8 +307,8 @@ func AttachFilesToRanges( }) if rg == nil { log.Panic("range not found", - zap.Stringer("startKey", utils.WrapKey(f.GetStartKey())), - zap.Stringer("endKey", utils.WrapKey(f.GetEndKey()))) + zap.Stringer("startKey", logutil.WrapKey(f.GetStartKey())), + zap.Stringer("endKey", logutil.WrapKey(f.GetEndKey()))) } file := *f rg.Files = append(rg.Files, &file) @@ -330,7 +331,7 @@ func ValidateFileRewriteRule(file *backup.File, rewriteRules *RewriteRules) erro log.Error( "cannot find rewrite rule for file start key", zap.Int64("tableID", tableID), - utils.ZapFile(file), + logutil.File(file), ) return errors.Annotate(berrors.ErrRestoreInvalidRewrite, "cannot find rewrite rule") } @@ -341,7 +342,7 @@ func ValidateFileRewriteRule(file *backup.File, rewriteRules *RewriteRules) erro log.Error( "cannot find rewrite rule for file end key", zap.Int64("tableID", tableID), - utils.ZapFile(file), + logutil.File(file), ) return errors.Annotate(berrors.ErrRestoreInvalidRewrite, "cannot find rewrite rule") } @@ -355,7 +356,7 @@ func ValidateFileRewriteRule(file *backup.File, rewriteRules *RewriteRules) erro zap.Int64("endTableID", endTableID), zap.Stringer("startRule", startRule), zap.Stringer("endRule", endRule), - utils.ZapFile(file), + logutil.File(file), ) return errors.Annotate(berrors.ErrRestoreInvalidRewrite, "unexpected rewrite rules") } @@ -442,7 +443,7 @@ func rewriteFileKeys(file *backup.File, rewriteRules *RewriteRules) (startKey, e startKey, rule = rewriteRawKey(file.GetStartKey(), rewriteRules) if rewriteRules != nil && rule == nil { log.Error("cannot find rewrite rule", - zap.Stringer("startKey", utils.WrapKey(file.GetStartKey())), + zap.Stringer("startKey", logutil.WrapKey(file.GetStartKey())), zap.Reflect("rewrite table", rewriteRules.Table), zap.Reflect("rewrite data", rewriteRules.Data)) err = errors.Annotate(berrors.ErrRestoreInvalidRewrite, "cannot find rewrite rule for start key") @@ -457,8 +458,8 @@ func rewriteFileKeys(file *backup.File, rewriteRules *RewriteRules) (startKey, e log.Error("table ids dont matched", zap.Int64("startID", startID), zap.Int64("endID", endID), - zap.Stringer("startKey", utils.WrapKey(startKey)), - zap.Stringer("endKey", utils.WrapKey(endKey))) + zap.Stringer("startKey", logutil.WrapKey(startKey)), + zap.Stringer("endKey", logutil.WrapKey(endKey))) err = errors.Annotate(berrors.ErrRestoreInvalidRewrite, "invalid table id") } return diff --git a/pkg/rtree/rtree.go b/pkg/rtree/rtree.go index aa8300892..4313eaa23 100644 --- a/pkg/rtree/rtree.go +++ b/pkg/rtree/rtree.go @@ -11,7 +11,7 @@ import ( "github.com/pingcap/log" "go.uber.org/zap" - "github.com/pingcap/br/pkg/utils" + "github.com/pingcap/br/pkg/logutil" ) // Range represents a backup response. @@ -134,8 +134,8 @@ func (rangeTree *RangeTree) Update(rg Range) { // Range has backuped, overwrite overlapping range. for _, item := range overlaps { log.Info("delete overlapping range", - zap.Stringer("StartKey", utils.WrapKey(item.StartKey)), - zap.Stringer("EndKey", utils.WrapKey(item.EndKey)), + zap.Stringer("StartKey", logutil.WrapKey(item.StartKey)), + zap.Stringer("EndKey", logutil.WrapKey(item.EndKey)), ) rangeTree.Delete(item) }