From 47ac524919aa5c084b81c5f933fd30b9b9352b72 Mon Sep 17 00:00:00 2001 From: Lonng Date: Sat, 16 Feb 2019 20:41:27 +0800 Subject: [PATCH 01/11] restore: write index kvs and data kvs to seperate engine --- lightning/restore/restore.go | 155 +++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 33 deletions(-) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index e203e6042..eb28f1856 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -31,6 +31,8 @@ import ( "github.com/cznic/mathutil" sstpb "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" "github.com/pingcap/tidb-lightning/lightning/kv" @@ -529,15 +531,16 @@ func (t *TableRestore) restoreTable( defer wg.Done() tag := fmt.Sprintf("%s:%d", t.tableName, eid) - closedEngine, closedEngineWorker, err := t.restoreEngine(ctx, rc, eid, ecp) + info, err := t.restoreEngine(ctx, rc, eid, ecp) rc.tableWorkers.Recycle(w) if err != nil { engineErr.Set(tag, err) return } - defer rc.closedEngineLimit.Recycle(closedEngineWorker) - if err := t.importEngine(ctx, closedEngine, rc, eid, ecp); err != nil { + defer rc.closedEngineLimit.Recycle(info.dataWorker) + defer rc.closedEngineLimit.Recycle(info.indexWorker) + if err := t.importEngine(ctx, info.dataEngine, info.indexEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } }(restoreWorker, engineID, engine) @@ -558,28 +561,60 @@ func (t *TableRestore) restoreTable( return errors.Trace(t.postProcess(ctx, rc, cp)) } +type closedEngineInfo struct { + dataEngine *kv.ClosedEngine + dataWorker *worker.Worker + indexEngine *kv.ClosedEngine + indexWorker *worker.Worker +} + func (t *TableRestore) restoreEngine( ctx context.Context, rc *RestoreController, engineID int, cp *EngineCheckpoint, -) (*kv.ClosedEngine, *worker.Worker, error) { +) (*closedEngineInfo, error) { if cp.Status >= CheckpointStatusClosed { - w := rc.closedEngineLimit.Apply() - closedEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) + dataWorker := rc.closedEngineLimit.Apply() + dataEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) // If any error occurred, recycle worker immediately if err != nil { - rc.closedEngineLimit.Recycle(w) - return closedEngine, nil, errors.Trace(err) + rc.closedEngineLimit.Recycle(dataWorker) + info := &closedEngineInfo{ + dataEngine: dataEngine, + } + return info, errors.Trace(err) + } + indexWorker := rc.closedEngineLimit.Apply() + indexEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, 1<<32+engineID) + if err != nil { + rc.closedEngineLimit.Recycle(dataWorker) + rc.closedEngineLimit.Recycle(indexWorker) + info := &closedEngineInfo{ + dataEngine: dataEngine, + indexEngine: indexEngine, + } + return info, errors.Trace(err) } - return closedEngine, w, nil + info := &closedEngineInfo{ + dataEngine: dataEngine, + dataWorker: dataWorker, + indexEngine: indexEngine, + indexWorker: indexWorker, + } + return info, nil } timer := time.Now() - engine, err := rc.importer.OpenEngine(ctx, t.tableName, engineID) + dataEngine, err := rc.importer.OpenEngine(ctx, t.tableName, engineID) + if err != nil { + return nil, errors.Trace(err) + } + + indexEngine, err := rc.importer.OpenEngine(ctx, t.tableName, 1<<32+engineID) if err != nil { - return nil, nil, errors.Trace(err) + return nil, errors.Trace(err) } var wg sync.WaitGroup @@ -593,7 +628,7 @@ func (t *TableRestore) restoreEngine( select { case <-ctx.Done(): - return nil, nil, ctx.Err() + return nil, ctx.Err() default: } @@ -609,7 +644,7 @@ func (t *TableRestore) restoreEngine( cr, err := newChunkRestore(chunkIndex, chunk, rc.cfg.Mydumper.ReadBlockSize, rc.ioWorkers) if err != nil { - return nil, nil, errors.Trace(err) + return nil, errors.Trace(err) } metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending).Inc() @@ -623,7 +658,7 @@ func (t *TableRestore) restoreEngine( rc.regionWorkers.Recycle(w) }() metric.ChunkCounter.WithLabelValues(metric.ChunkStateRunning).Inc() - err := cr.restore(ctx, t, engineID, engine, rc) + err := cr.restore(ctx, t, engineID, dataEngine, indexEngine, rc) if err == nil { metric.ChunkCounter.WithLabelValues(metric.ChunkStateFinished).Inc() return @@ -649,24 +684,40 @@ func (t *TableRestore) restoreEngine( err = chunkErr.Get() rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusAllWritten) if err != nil { - return nil, nil, errors.Trace(err) + return nil, errors.Trace(err) } - w := rc.closedEngineLimit.Apply() - closedEngine, err := engine.Close(ctx) + dataWorker := rc.closedEngineLimit.Apply() + closedDataEngine, err := dataEngine.Close(ctx) rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusClosed) if err != nil { common.AppLogger.Errorf("[kv-deliver] flush stage with error (step = close) : %s", errors.ErrorStack(err)) // If any error occurred, recycle worker immediately - rc.closedEngineLimit.Recycle(w) - return nil, nil, errors.Trace(err) + rc.closedEngineLimit.Recycle(dataWorker) + return nil, errors.Trace(err) + } + indexWorker := rc.closedEngineLimit.Apply() + closedIndexEngine, err := indexEngine.Close(ctx) + if err != nil { + common.AppLogger.Errorf("[kv-deliver] flush stage with error (step = close) : %s", errors.ErrorStack(err)) + rc.closedEngineLimit.Recycle(dataWorker) + rc.closedEngineLimit.Recycle(indexWorker) + return nil, errors.Trace(err) } - return closedEngine, w, nil + + info := &closedEngineInfo{ + dataEngine: closedDataEngine, + dataWorker: dataWorker, + indexEngine: closedIndexEngine, + indexWorker: indexWorker, + } + return info, nil } func (t *TableRestore) importEngine( ctx context.Context, - closedEngine *kv.ClosedEngine, + dataEngine *kv.ClosedEngine, + indexEngine *kv.ClosedEngine, rc *RestoreController, engineID int, cp *EngineCheckpoint, @@ -680,7 +731,7 @@ func (t *TableRestore) importEngine( // the lock ensures the import() step will not be concurrent. rc.postProcessLock.Lock() - err := t.importKV(ctx, closedEngine) + err := t.importKV(ctx, dataEngine, indexEngine) // gofail: var SlowDownImport struct{} rc.postProcessLock.Unlock() rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusImported) @@ -1055,19 +1106,25 @@ func (tr *TableRestore) restoreTableMeta(ctx context.Context, db *sql.DB) error return nil } -func (tr *TableRestore) importKV(ctx context.Context, closedEngine *kv.ClosedEngine) error { +func (tr *TableRestore) importKV(ctx context.Context, dataEngine, indexEngine *kv.ClosedEngine) error { common.AppLogger.Infof("[%s] flush kv deliver ...", tr.tableName) - start := time.Now() - err := closedEngine.Import(ctx) - if err != nil { + if err := dataEngine.Import(ctx); err != nil { + if !common.IsContextCanceledError(err) { + common.AppLogger.Errorf("[%s] failed to flush kvs : %s", tr.tableName, err.Error()) + } + return errors.Trace(err) + } + dataEngine.Cleanup(ctx) + + if err := indexEngine.Import(ctx); err != nil { if !common.IsContextCanceledError(err) { common.AppLogger.Errorf("[%s] failed to flush kvs : %s", tr.tableName, err.Error()) } return errors.Trace(err) } - closedEngine.Cleanup(ctx) + indexEngine.Cleanup(ctx) dur := time.Since(start) metric.ImportSecondsHistogram.Observe(dur.Seconds()) @@ -1241,7 +1298,7 @@ func (cr *chunkRestore) restore( ctx context.Context, t *TableRestore, engineID int, - engine *kv.OpenedEngine, + dataEngine, indexEngine *kv.OpenedEngine, rc *RestoreController, ) error { // Create the encoder. @@ -1280,6 +1337,7 @@ func (cr *chunkRestore) restore( deliverCompleteCh := make(chan error, 1) go func() { + var dataKVs, indexKVs []kvenc.KvPair for { block.cond.L.Lock() for !block.encodeCompleted && len(block.totalKVs) == 0 { @@ -1297,14 +1355,28 @@ func (cr *chunkRestore) restore( // kv -> deliver ( -> tikv ) start := time.Now() - stream, err := engine.NewWriteStream(ctx) + dataStream, err := dataEngine.NewWriteStream(ctx) if err != nil { deliverCompleteCh <- errors.Trace(err) return } + indexStream, err := indexEngine.NewWriteStream(ctx) + if err != nil { + deliverCompleteCh <- errors.Trace(err) + return + } + // class kvs + for _, k := range b.totalKVs { + if k.Key[tablecodec.TableSplitKeyLen+1] == 'r' { + dataKVs = append(dataKVs, k) + } else { + indexKVs = append(indexKVs, k) + } + } + b.totalKVs = nil - for _, kvs := range splitIntoDeliveryStreams(b.totalKVs, maxDeliverBytes) { - if e := stream.Put(kvs); e != nil { + for _, kvs := range splitIntoDeliveryStreams(dataKVs, maxDeliverBytes) { + if e := dataStream.Put(kvs); e != nil { if err != nil { common.AppLogger.Warnf("failed to put write stream: %s", e.Error()) } else { @@ -1312,10 +1384,27 @@ func (cr *chunkRestore) restore( } } } - b.totalKVs = nil + for _, kvs := range splitIntoDeliveryStreams(indexKVs, maxDeliverBytes) { + if e := indexStream.Put(kvs); e != nil { + if err != nil { + common.AppLogger.Warnf("failed to put write stream: %s", e.Error()) + } else { + err = e + } + } + } + dataKVs = dataKVs[:0] + indexKVs = indexKVs[:0] block.cond.Signal() - if e := stream.Close(); e != nil { + if e := dataStream.Close(); e != nil { + if err != nil { + common.AppLogger.Warnf("[%s:%d] failed to close write stream: %s", t.tableName, engineID, e.Error()) + } else { + err = e + } + } + if e := indexStream.Close(); e != nil { if err != nil { common.AppLogger.Warnf("[%s:%d] failed to close write stream: %s", t.tableName, engineID, e.Error()) } else { From d8335d395e0cd231ad0122cdccb57382057343d2 Mon Sep 17 00:00:00 2001 From: Lonng Date: Mon, 18 Feb 2019 10:09:09 +0800 Subject: [PATCH 02/11] restore: rename to --- lightning/restore/restore.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index eb28f1856..1ca5c4ed7 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -531,16 +531,16 @@ func (t *TableRestore) restoreTable( defer wg.Done() tag := fmt.Sprintf("%s:%d", t.tableName, eid) - info, err := t.restoreEngine(ctx, rc, eid, ecp) + result, err := t.restoreEngine(ctx, rc, eid, ecp) rc.tableWorkers.Recycle(w) if err != nil { engineErr.Set(tag, err) return } - defer rc.closedEngineLimit.Recycle(info.dataWorker) - defer rc.closedEngineLimit.Recycle(info.indexWorker) - if err := t.importEngine(ctx, info.dataEngine, info.indexEngine, rc, eid, ecp); err != nil { + defer rc.closedEngineLimit.Recycle(result.dataWorker) + defer rc.closedEngineLimit.Recycle(result.indexWorker) + if err := t.importEngine(ctx, result.dataEngine, result.indexEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } }(restoreWorker, engineID, engine) @@ -561,7 +561,7 @@ func (t *TableRestore) restoreTable( return errors.Trace(t.postProcess(ctx, rc, cp)) } -type closedEngineInfo struct { +type restoreResult struct { dataEngine *kv.ClosedEngine dataWorker *worker.Worker indexEngine *kv.ClosedEngine @@ -573,30 +573,30 @@ func (t *TableRestore) restoreEngine( rc *RestoreController, engineID int, cp *EngineCheckpoint, -) (*closedEngineInfo, error) { +) (*restoreResult, error) { if cp.Status >= CheckpointStatusClosed { dataWorker := rc.closedEngineLimit.Apply() dataEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) // If any error occurred, recycle worker immediately if err != nil { rc.closedEngineLimit.Recycle(dataWorker) - info := &closedEngineInfo{ + result := &restoreResult{ dataEngine: dataEngine, } - return info, errors.Trace(err) + return result, errors.Trace(err) } indexWorker := rc.closedEngineLimit.Apply() indexEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, 1<<32+engineID) if err != nil { rc.closedEngineLimit.Recycle(dataWorker) rc.closedEngineLimit.Recycle(indexWorker) - info := &closedEngineInfo{ + info := &restoreResult{ dataEngine: dataEngine, indexEngine: indexEngine, } return info, errors.Trace(err) } - info := &closedEngineInfo{ + info := &restoreResult{ dataEngine: dataEngine, dataWorker: dataWorker, indexEngine: indexEngine, @@ -705,13 +705,13 @@ func (t *TableRestore) restoreEngine( return nil, errors.Trace(err) } - info := &closedEngineInfo{ + result := &restoreResult{ dataEngine: closedDataEngine, dataWorker: dataWorker, indexEngine: closedIndexEngine, indexWorker: indexWorker, } - return info, nil + return result, nil } func (t *TableRestore) importEngine( From 89c628d4d075118f27d39a47ed44fffa16892460 Mon Sep 17 00:00:00 2001 From: Lonng Date: Mon, 18 Feb 2019 11:57:59 +0800 Subject: [PATCH 03/11] restore: use single engine file to store index --- lightning/restore/restore.go | 95 ++++++++++++++---------------------- 1 file changed, 36 insertions(+), 59 deletions(-) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 1ca5c4ed7..57bc93f0b 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -503,6 +503,10 @@ func (t *TableRestore) restoreTable( } // 2. Restore engines (if still needed) + indexEngine, err := rc.importer.OpenEngine(ctx, t.tableName, -1) + if err != nil { + return errors.Trace(err) + } if cp.Status < CheckpointStatusImported { timer := time.Now() @@ -531,7 +535,7 @@ func (t *TableRestore) restoreTable( defer wg.Done() tag := fmt.Sprintf("%s:%d", t.tableName, eid) - result, err := t.restoreEngine(ctx, rc, eid, ecp) + result, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) rc.tableWorkers.Recycle(w) if err != nil { engineErr.Set(tag, err) @@ -539,8 +543,7 @@ func (t *TableRestore) restoreTable( } defer rc.closedEngineLimit.Recycle(result.dataWorker) - defer rc.closedEngineLimit.Recycle(result.indexWorker) - if err := t.importEngine(ctx, result.dataEngine, result.indexEngine, rc, eid, ecp); err != nil { + if err := t.importEngine(ctx, result.closedEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } }(restoreWorker, engineID, engine) @@ -557,50 +560,40 @@ func (t *TableRestore) restoreTable( } // 3. Post-process - - return errors.Trace(t.postProcess(ctx, rc, cp)) + closedIndexEngine, err := indexEngine.Close(ctx) + if err != nil { + common.AppLogger.Errorf("[kv-deliver] index engine closed error: %s", errors.ErrorStack(err)) + return errors.Trace(err) + } + return errors.Trace(t.postProcess(ctx, rc, cp, closedIndexEngine)) } type restoreResult struct { - dataEngine *kv.ClosedEngine - dataWorker *worker.Worker - indexEngine *kv.ClosedEngine - indexWorker *worker.Worker + closedEngine *kv.ClosedEngine + dataWorker *worker.Worker } func (t *TableRestore) restoreEngine( ctx context.Context, rc *RestoreController, + indexEngine *kv.OpenedEngine, engineID int, cp *EngineCheckpoint, ) (*restoreResult, error) { if cp.Status >= CheckpointStatusClosed { dataWorker := rc.closedEngineLimit.Apply() - dataEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) + closedEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) // If any error occurred, recycle worker immediately if err != nil { rc.closedEngineLimit.Recycle(dataWorker) result := &restoreResult{ - dataEngine: dataEngine, + closedEngine: closedEngine, } return result, errors.Trace(err) } - indexWorker := rc.closedEngineLimit.Apply() - indexEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, 1<<32+engineID) - if err != nil { - rc.closedEngineLimit.Recycle(dataWorker) - rc.closedEngineLimit.Recycle(indexWorker) - info := &restoreResult{ - dataEngine: dataEngine, - indexEngine: indexEngine, - } - return info, errors.Trace(err) - } info := &restoreResult{ - dataEngine: dataEngine, - dataWorker: dataWorker, - indexEngine: indexEngine, - indexWorker: indexWorker, + closedEngine: closedEngine, + dataWorker: dataWorker, } return info, nil } @@ -612,11 +605,6 @@ func (t *TableRestore) restoreEngine( return nil, errors.Trace(err) } - indexEngine, err := rc.importer.OpenEngine(ctx, t.tableName, 1<<32+engineID) - if err != nil { - return nil, errors.Trace(err) - } - var wg sync.WaitGroup var chunkErr common.OnceError @@ -696,28 +684,16 @@ func (t *TableRestore) restoreEngine( rc.closedEngineLimit.Recycle(dataWorker) return nil, errors.Trace(err) } - indexWorker := rc.closedEngineLimit.Apply() - closedIndexEngine, err := indexEngine.Close(ctx) - if err != nil { - common.AppLogger.Errorf("[kv-deliver] flush stage with error (step = close) : %s", errors.ErrorStack(err)) - rc.closedEngineLimit.Recycle(dataWorker) - rc.closedEngineLimit.Recycle(indexWorker) - return nil, errors.Trace(err) - } - result := &restoreResult{ - dataEngine: closedDataEngine, - dataWorker: dataWorker, - indexEngine: closedIndexEngine, - indexWorker: indexWorker, + closedEngine: closedDataEngine, + dataWorker: dataWorker, } return result, nil } func (t *TableRestore) importEngine( ctx context.Context, - dataEngine *kv.ClosedEngine, - indexEngine *kv.ClosedEngine, + closedEngine *kv.ClosedEngine, rc *RestoreController, engineID int, cp *EngineCheckpoint, @@ -731,7 +707,7 @@ func (t *TableRestore) importEngine( // the lock ensures the import() step will not be concurrent. rc.postProcessLock.Lock() - err := t.importKV(ctx, dataEngine, indexEngine) + err := t.importKV(ctx, closedEngine) // gofail: var SlowDownImport struct{} rc.postProcessLock.Unlock() rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusImported) @@ -755,7 +731,16 @@ func (t *TableRestore) importEngine( return nil } -func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint) error { +func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint, indexEngine *kv.ClosedEngine) error { + // the lock ensures the import() step will not be concurrent. + rc.postProcessLock.Lock() + err := t.importKV(ctx, indexEngine) + rc.postProcessLock.Unlock() + if err != nil { + common.AppLogger.Errorf("[%[1]s] failed to import index engine: %v", t.tableName, err.Error()) + return errors.Trace(err) + } + setSessionConcurrencyVars(ctx, rc.tidbMgr.db, rc.cfg.TiDB) // 3. alter table set auto_increment @@ -1106,25 +1091,17 @@ func (tr *TableRestore) restoreTableMeta(ctx context.Context, db *sql.DB) error return nil } -func (tr *TableRestore) importKV(ctx context.Context, dataEngine, indexEngine *kv.ClosedEngine) error { +func (tr *TableRestore) importKV(ctx context.Context, closedEngine *kv.ClosedEngine) error { common.AppLogger.Infof("[%s] flush kv deliver ...", tr.tableName) start := time.Now() - if err := dataEngine.Import(ctx); err != nil { - if !common.IsContextCanceledError(err) { - common.AppLogger.Errorf("[%s] failed to flush kvs : %s", tr.tableName, err.Error()) - } - return errors.Trace(err) - } - dataEngine.Cleanup(ctx) - - if err := indexEngine.Import(ctx); err != nil { + if err := closedEngine.Import(ctx); err != nil { if !common.IsContextCanceledError(err) { common.AppLogger.Errorf("[%s] failed to flush kvs : %s", tr.tableName, err.Error()) } return errors.Trace(err) } - indexEngine.Cleanup(ctx) + closedEngine.Cleanup(ctx) dur := time.Since(start) metric.ImportSecondsHistogram.Observe(dur.Seconds()) From fc953dcaf62d30b7edbe2a5d08f680cf19d34dc0 Mon Sep 17 00:00:00 2001 From: Lonng Date: Wed, 20 Feb 2019 17:39:08 +0800 Subject: [PATCH 04/11] restore: make index engine limited by table concurrency --- lightning/restore/restore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 57bc93f0b..8ce06ff68 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -503,6 +503,8 @@ func (t *TableRestore) restoreTable( } // 2. Restore engines (if still needed) + indexWorker := rc.tableWorkers.Apply() + defer rc.tableWorkers.Recycle(indexWorker) indexEngine, err := rc.importer.OpenEngine(ctx, t.tableName, -1) if err != nil { return errors.Trace(err) From 26e147f22ee35f4f7f7acf5d91d5975d2102910d Mon Sep 17 00:00:00 2001 From: Lonng Date: Mon, 4 Mar 2019 13:24:56 +0800 Subject: [PATCH 05/11] restore: revert some changes --- lightning/restore/restore.go | 44 ++++++++++++------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 8ce06ff68..43da4c299 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -537,15 +537,15 @@ func (t *TableRestore) restoreTable( defer wg.Done() tag := fmt.Sprintf("%s:%d", t.tableName, eid) - result, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) + dataClosedEngine, dataWorker, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) rc.tableWorkers.Recycle(w) if err != nil { engineErr.Set(tag, err) return } - defer rc.closedEngineLimit.Recycle(result.dataWorker) - if err := t.importEngine(ctx, result.closedEngine, rc, eid, ecp); err != nil { + defer rc.closedEngineLimit.Recycle(dataWorker) + if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } }(restoreWorker, engineID, engine) @@ -570,41 +570,29 @@ func (t *TableRestore) restoreTable( return errors.Trace(t.postProcess(ctx, rc, cp, closedIndexEngine)) } -type restoreResult struct { - closedEngine *kv.ClosedEngine - dataWorker *worker.Worker -} - func (t *TableRestore) restoreEngine( ctx context.Context, rc *RestoreController, indexEngine *kv.OpenedEngine, engineID int, cp *EngineCheckpoint, -) (*restoreResult, error) { +) (*kv.ClosedEngine, *worker.Worker, error) { if cp.Status >= CheckpointStatusClosed { - dataWorker := rc.closedEngineLimit.Apply() + w := rc.closedEngineLimit.Apply() closedEngine, err := rc.importer.UnsafeCloseEngine(ctx, t.tableName, engineID) // If any error occurred, recycle worker immediately if err != nil { - rc.closedEngineLimit.Recycle(dataWorker) - result := &restoreResult{ - closedEngine: closedEngine, - } - return result, errors.Trace(err) + rc.closedEngineLimit.Recycle(w) + return closedEngine, nil, errors.Trace(err) } - info := &restoreResult{ - closedEngine: closedEngine, - dataWorker: dataWorker, - } - return info, nil + return closedEngine, w, nil } timer := time.Now() dataEngine, err := rc.importer.OpenEngine(ctx, t.tableName, engineID) if err != nil { - return nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } var wg sync.WaitGroup @@ -618,7 +606,7 @@ func (t *TableRestore) restoreEngine( select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, nil, ctx.Err() default: } @@ -634,7 +622,7 @@ func (t *TableRestore) restoreEngine( cr, err := newChunkRestore(chunkIndex, chunk, rc.cfg.Mydumper.ReadBlockSize, rc.ioWorkers) if err != nil { - return nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending).Inc() @@ -674,7 +662,7 @@ func (t *TableRestore) restoreEngine( err = chunkErr.Get() rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusAllWritten) if err != nil { - return nil, errors.Trace(err) + return nil, nil, errors.Trace(err) } dataWorker := rc.closedEngineLimit.Apply() @@ -684,13 +672,9 @@ func (t *TableRestore) restoreEngine( common.AppLogger.Errorf("[kv-deliver] flush stage with error (step = close) : %s", errors.ErrorStack(err)) // If any error occurred, recycle worker immediately rc.closedEngineLimit.Recycle(dataWorker) - return nil, errors.Trace(err) - } - result := &restoreResult{ - closedEngine: closedDataEngine, - dataWorker: dataWorker, + return nil, nil, errors.Trace(err) } - return result, nil + return closedDataEngine, dataWorker, nil } func (t *TableRestore) importEngine( From 85fcd71926cf852c2ab9cab87c28c4a6eed4c272 Mon Sep 17 00:00:00 2001 From: Lonng Date: Mon, 4 Mar 2019 14:45:33 +0800 Subject: [PATCH 06/11] restore: modify checkpoint proto --- lightning/restore/checkpoints.go | 4 +- lightning/restore/file_checkpoints.pb.go | 390 ++++++++++++++++------- lightning/restore/file_checkpoints.proto | 1 + 3 files changed, 273 insertions(+), 122 deletions(-) diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index 8a706a869..abcb43e62 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -54,7 +54,7 @@ const ( // the table names to store each kind of checkpoint in the checkpoint database // remember to increase the version number in case of incompatible change. checkpointTableNameTable = "table_v4" - checkpointTableNameEngine = "engine_v4" + checkpointTableNameEngine = "engine_v5" checkpointTableNameChunk = "chunk_v4" ) @@ -318,7 +318,7 @@ func NewMySQLCheckpointsDB(ctx context.Context, db *sql.DB, schemaName string) ( err = common.ExecWithRetry(ctx, db, "(create engine checkpoints table)", fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s.%s ( table_name varchar(261) NOT NULL, - engine_id int unsigned NOT NULL, + engine_id int NOT NULL, status tinyint unsigned DEFAULT 30, create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index a12a7284d..cf0ab9c0b 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -3,14 +3,14 @@ package restore -import proto "github.com/gogo/protobuf/proto" -import fmt "fmt" -import math "math" -import _ "github.com/gogo/protobuf/gogoproto" - -import encoding_binary "encoding/binary" - -import io "io" +import ( + encoding_binary "encoding/binary" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" +) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -25,16 +25,14 @@ const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package type CheckpointsModel struct { // key is table_name - Checkpoints map[string]*TableCheckpointModel `protobuf:"bytes,1,rep,name=checkpoints" json:"checkpoints,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Checkpoints map[string]*TableCheckpointModel `protobuf:"bytes,1,rep,name=checkpoints,proto3" json:"checkpoints,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *CheckpointsModel) Reset() { *m = CheckpointsModel{} } func (m *CheckpointsModel) String() string { return proto.CompactTextString(m) } func (*CheckpointsModel) ProtoMessage() {} func (*CheckpointsModel) Descriptor() ([]byte, []int) { - return fileDescriptor_file_checkpoints_168275cfec5db5bf, []int{0} + return fileDescriptor_c47ec4f2f281cd62, []int{0} } func (m *CheckpointsModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -51,8 +49,8 @@ func (m *CheckpointsModel) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return b[:n], nil } } -func (dst *CheckpointsModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_CheckpointsModel.Merge(dst, src) +func (m *CheckpointsModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_CheckpointsModel.Merge(m, src) } func (m *CheckpointsModel) XXX_Size() int { return m.Size() @@ -64,19 +62,18 @@ func (m *CheckpointsModel) XXX_DiscardUnknown() { var xxx_messageInfo_CheckpointsModel proto.InternalMessageInfo type TableCheckpointModel struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` - Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` - AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` - Engines []*EngineCheckpointModel `protobuf:"bytes,6,rep,name=engines" json:"engines,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` + AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` + Engines []*EngineCheckpointModel `protobuf:"bytes,6,rep,name=engines,proto3" json:"engines,omitempty"` + IndexEngineStatuses []uint32 `protobuf:"varint,7,rep,packed,name=index_engine_statuses,json=indexEngineStatuses,proto3" json:"index_engine_statuses,omitempty"` } func (m *TableCheckpointModel) Reset() { *m = TableCheckpointModel{} } func (m *TableCheckpointModel) String() string { return proto.CompactTextString(m) } func (*TableCheckpointModel) ProtoMessage() {} func (*TableCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_file_checkpoints_168275cfec5db5bf, []int{1} + return fileDescriptor_c47ec4f2f281cd62, []int{1} } func (m *TableCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -93,8 +90,8 @@ func (m *TableCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byte return b[:n], nil } } -func (dst *TableCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_TableCheckpointModel.Merge(dst, src) +func (m *TableCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_TableCheckpointModel.Merge(m, src) } func (m *TableCheckpointModel) XXX_Size() int { return m.Size() @@ -108,16 +105,14 @@ var xxx_messageInfo_TableCheckpointModel proto.InternalMessageInfo type EngineCheckpointModel struct { Status uint32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // key is "$path:$offset" - Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *EngineCheckpointModel) Reset() { *m = EngineCheckpointModel{} } func (m *EngineCheckpointModel) String() string { return proto.CompactTextString(m) } func (*EngineCheckpointModel) ProtoMessage() {} func (*EngineCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_file_checkpoints_168275cfec5db5bf, []int{2} + return fileDescriptor_c47ec4f2f281cd62, []int{2} } func (m *EngineCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -134,8 +129,8 @@ func (m *EngineCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byt return b[:n], nil } } -func (dst *EngineCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_EngineCheckpointModel.Merge(dst, src) +func (m *EngineCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_EngineCheckpointModel.Merge(m, src) } func (m *EngineCheckpointModel) XXX_Size() int { return m.Size() @@ -147,26 +142,24 @@ func (m *EngineCheckpointModel) XXX_DiscardUnknown() { var xxx_messageInfo_EngineCheckpointModel proto.InternalMessageInfo type ChunkCheckpointModel struct { - Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` - Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` - Columns []byte `protobuf:"bytes,3,opt,name=columns,proto3" json:"columns,omitempty"` - ShouldIncludeRowId bool `protobuf:"varint,4,opt,name=should_include_row_id,json=shouldIncludeRowId,proto3" json:"should_include_row_id,omitempty"` - EndOffset int64 `protobuf:"varint,5,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` - Pos int64 `protobuf:"varint,6,opt,name=pos,proto3" json:"pos,omitempty"` - PrevRowidMax int64 `protobuf:"varint,7,opt,name=prev_rowid_max,json=prevRowidMax,proto3" json:"prev_rowid_max,omitempty"` - RowidMax int64 `protobuf:"varint,8,opt,name=rowid_max,json=rowidMax,proto3" json:"rowid_max,omitempty"` - KvcBytes uint64 `protobuf:"varint,9,opt,name=kvc_bytes,json=kvcBytes,proto3" json:"kvc_bytes,omitempty"` - KvcKvs uint64 `protobuf:"varint,10,opt,name=kvc_kvs,json=kvcKvs,proto3" json:"kvc_kvs,omitempty"` - KvcChecksum uint64 `protobuf:"fixed64,11,opt,name=kvc_checksum,json=kvcChecksum,proto3" json:"kvc_checksum,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_sizecache int32 `json:"-"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + Columns []byte `protobuf:"bytes,3,opt,name=columns,proto3" json:"columns,omitempty"` + ShouldIncludeRowId bool `protobuf:"varint,4,opt,name=should_include_row_id,json=shouldIncludeRowId,proto3" json:"should_include_row_id,omitempty"` + EndOffset int64 `protobuf:"varint,5,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` + Pos int64 `protobuf:"varint,6,opt,name=pos,proto3" json:"pos,omitempty"` + PrevRowidMax int64 `protobuf:"varint,7,opt,name=prev_rowid_max,json=prevRowidMax,proto3" json:"prev_rowid_max,omitempty"` + RowidMax int64 `protobuf:"varint,8,opt,name=rowid_max,json=rowidMax,proto3" json:"rowid_max,omitempty"` + KvcBytes uint64 `protobuf:"varint,9,opt,name=kvc_bytes,json=kvcBytes,proto3" json:"kvc_bytes,omitempty"` + KvcKvs uint64 `protobuf:"varint,10,opt,name=kvc_kvs,json=kvcKvs,proto3" json:"kvc_kvs,omitempty"` + KvcChecksum uint64 `protobuf:"fixed64,11,opt,name=kvc_checksum,json=kvcChecksum,proto3" json:"kvc_checksum,omitempty"` } func (m *ChunkCheckpointModel) Reset() { *m = ChunkCheckpointModel{} } func (m *ChunkCheckpointModel) String() string { return proto.CompactTextString(m) } func (*ChunkCheckpointModel) ProtoMessage() {} func (*ChunkCheckpointModel) Descriptor() ([]byte, []int) { - return fileDescriptor_file_checkpoints_168275cfec5db5bf, []int{3} + return fileDescriptor_c47ec4f2f281cd62, []int{3} } func (m *ChunkCheckpointModel) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -183,8 +176,8 @@ func (m *ChunkCheckpointModel) XXX_Marshal(b []byte, deterministic bool) ([]byte return b[:n], nil } } -func (dst *ChunkCheckpointModel) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunkCheckpointModel.Merge(dst, src) +func (m *ChunkCheckpointModel) XXX_Merge(src proto.Message) { + xxx_messageInfo_ChunkCheckpointModel.Merge(m, src) } func (m *ChunkCheckpointModel) XXX_Size() int { return m.Size() @@ -203,6 +196,52 @@ func init() { proto.RegisterMapType((map[string]*ChunkCheckpointModel)(nil), "EngineCheckpointModel.ChunksEntry") proto.RegisterType((*ChunkCheckpointModel)(nil), "ChunkCheckpointModel") } + +func init() { + proto.RegisterFile("lightning/restore/file_checkpoints.proto", fileDescriptor_c47ec4f2f281cd62) +} + +var fileDescriptor_c47ec4f2f281cd62 = []byte{ + // 583 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcb, 0x6e, 0xd3, 0x4c, + 0x14, 0xce, 0xc4, 0x6d, 0x2e, 0x27, 0xe9, 0xaf, 0x6a, 0xfe, 0xa6, 0x8c, 0x8a, 0xb0, 0x4c, 0x04, + 0x92, 0x25, 0x84, 0x03, 0x65, 0x83, 0xba, 0x6c, 0xe9, 0xa2, 0x42, 0x15, 0x68, 0x80, 0x0d, 0x1b, + 0xcb, 0x97, 0x89, 0x6d, 0xd9, 0x99, 0x89, 0x3c, 0xb6, 0xdb, 0xbe, 0x05, 0x6f, 0xc2, 0x13, 0xb0, + 0xa6, 0xcb, 0x2e, 0x59, 0xb0, 0x80, 0xf6, 0x45, 0xd0, 0x8c, 0x5d, 0xc5, 0xad, 0x22, 0xc4, 0xee, + 0x7c, 0x97, 0xf9, 0xce, 0x58, 0x9f, 0xc6, 0x60, 0x67, 0x49, 0x14, 0x17, 0x3c, 0xe1, 0xd1, 0x2c, + 0x67, 0xb2, 0x10, 0x39, 0x9b, 0xcd, 0x93, 0x8c, 0xb9, 0x41, 0xcc, 0x82, 0x74, 0x29, 0x12, 0x5e, + 0x48, 0x67, 0x99, 0x8b, 0x42, 0xec, 0x3d, 0x8f, 0x92, 0x22, 0x2e, 0x7d, 0x27, 0x10, 0x8b, 0x59, + 0x24, 0x22, 0x31, 0xd3, 0xb4, 0x5f, 0xce, 0x35, 0xd2, 0x40, 0x4f, 0xb5, 0x7d, 0xfa, 0x15, 0xc1, + 0xf6, 0xd1, 0x2a, 0xe4, 0x54, 0x84, 0x2c, 0xc3, 0x6f, 0x60, 0xd4, 0x0a, 0x26, 0xc8, 0x32, 0xec, + 0xd1, 0xfe, 0xd4, 0xb9, 0xef, 0x6b, 0x13, 0xc7, 0xbc, 0xc8, 0x2f, 0x68, 0xfb, 0xd8, 0xde, 0xa7, + 0x3b, 0xc9, 0xda, 0x80, 0xb7, 0xc1, 0x48, 0xd9, 0x05, 0x41, 0x16, 0xb2, 0x87, 0x54, 0x8d, 0xf8, + 0x19, 0x6c, 0x56, 0x5e, 0x56, 0x32, 0xd2, 0xb5, 0x90, 0x3d, 0xda, 0x9f, 0x38, 0x1f, 0x3d, 0x3f, + 0x63, 0xab, 0x83, 0x7a, 0x13, 0xad, 0x3d, 0x07, 0xdd, 0xd7, 0x68, 0xfa, 0x1d, 0xc1, 0xce, 0x3a, + 0x0f, 0xc6, 0xb0, 0x11, 0x7b, 0x32, 0xd6, 0xe1, 0x63, 0xaa, 0x67, 0xbc, 0x0b, 0x3d, 0x59, 0x78, + 0x45, 0x29, 0x89, 0x61, 0x21, 0x7b, 0x8b, 0x36, 0x08, 0x3f, 0x02, 0xf0, 0xb2, 0x4c, 0x04, 0xae, + 0xef, 0x49, 0x46, 0x36, 0x2c, 0x64, 0x1b, 0x74, 0xa8, 0x99, 0x43, 0x4f, 0x32, 0xfc, 0x02, 0xfa, + 0x8c, 0x47, 0x09, 0x67, 0x92, 0xf4, 0xf4, 0xc7, 0xef, 0x3a, 0xc7, 0x1a, 0xdf, 0xbf, 0xd7, 0xad, + 0x0d, 0xef, 0xc3, 0x24, 0xe1, 0x21, 0x3b, 0x77, 0x6b, 0xc2, 0xad, 0xf7, 0x30, 0x49, 0xfa, 0x96, + 0x61, 0x6f, 0xd1, 0xff, 0xb5, 0x58, 0x67, 0x7c, 0x68, 0xa4, 0xe9, 0x37, 0x04, 0x93, 0xb5, 0xb1, + 0xad, 0x6b, 0xa3, 0x3b, 0xd7, 0x3e, 0x80, 0x5e, 0x10, 0x97, 0x3c, 0x95, 0xa4, 0xdb, 0x74, 0xb2, + 0xf6, 0xbc, 0x73, 0xa4, 0x4d, 0x75, 0x27, 0xcd, 0x89, 0xbd, 0xf7, 0x30, 0x6a, 0xd1, 0xff, 0xd2, + 0x84, 0xb6, 0xff, 0xa5, 0x89, 0x9f, 0x5d, 0xd8, 0x59, 0xe7, 0x51, 0x4d, 0x2c, 0xbd, 0x22, 0x6e, + 0xc2, 0xf5, 0xac, 0x3e, 0x49, 0xcc, 0xe7, 0x92, 0x15, 0x3a, 0xde, 0xa0, 0x0d, 0xc2, 0x04, 0xfa, + 0x81, 0xc8, 0xca, 0x05, 0xaf, 0x2b, 0x1a, 0xd3, 0x5b, 0x88, 0x5f, 0xc2, 0x44, 0xc6, 0xa2, 0xcc, + 0x42, 0x37, 0xe1, 0x41, 0x56, 0x86, 0xcc, 0xcd, 0xc5, 0x99, 0x9b, 0x84, 0xba, 0xae, 0x01, 0xc5, + 0xb5, 0x78, 0x52, 0x6b, 0x54, 0x9c, 0x9d, 0x84, 0xaa, 0x56, 0xc6, 0x43, 0xb7, 0x59, 0xb4, 0x59, + 0xd7, 0xca, 0x78, 0xf8, 0xae, 0xde, 0xb5, 0x0d, 0xc6, 0x52, 0xa8, 0x4a, 0x15, 0xaf, 0x46, 0xfc, + 0x04, 0xfe, 0x5b, 0xe6, 0xac, 0x52, 0xc9, 0x49, 0xe8, 0x2e, 0xbc, 0x73, 0xd2, 0xd7, 0xe2, 0x58, + 0xb1, 0x54, 0x91, 0xa7, 0xde, 0x39, 0x7e, 0x08, 0xc3, 0x95, 0x61, 0xa0, 0x0d, 0x83, 0xbc, 0x25, + 0xa6, 0x55, 0xe0, 0xfa, 0x17, 0x05, 0x93, 0x64, 0x68, 0x21, 0x7b, 0x83, 0x0e, 0xd2, 0x2a, 0x38, + 0x54, 0x18, 0x3f, 0x80, 0xbe, 0x12, 0xd3, 0x4a, 0x12, 0xd0, 0x52, 0x2f, 0xad, 0x82, 0xb7, 0x95, + 0xc4, 0x8f, 0x61, 0xac, 0x04, 0xfd, 0x5e, 0x64, 0xb9, 0x20, 0x23, 0x0b, 0xd9, 0x3d, 0x3a, 0x4a, + 0xab, 0xe0, 0xa8, 0xa1, 0x0e, 0x9f, 0x5e, 0xfe, 0x36, 0x3b, 0x97, 0xd7, 0x26, 0xba, 0xba, 0x36, + 0xd1, 0xaf, 0x6b, 0x13, 0x7d, 0xb9, 0x31, 0x3b, 0x57, 0x37, 0x66, 0xe7, 0xc7, 0x8d, 0xd9, 0xf9, + 0xdc, 0x6f, 0xfe, 0x03, 0x7e, 0x4f, 0x3f, 0xe4, 0x57, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, + 0xf1, 0xe2, 0x3d, 0x23, 0x04, 0x00, 0x00, +} + func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -292,6 +331,23 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { i += n } } + if len(m.IndexEngineStatuses) > 0 { + dAtA3 := make([]byte, len(m.IndexEngineStatuses)*10) + var j2 int + for _, num := range m.IndexEngineStatuses { + for num >= 1<<7 { + dAtA3[j2] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j2++ + } + dAtA3[j2] = uint8(num) + j2++ + } + dAtA[i] = 0x3a + i++ + i = encodeVarintFileCheckpoints(dAtA, i, uint64(j2)) + i += copy(dAtA[i:], dAtA3[:j2]) + } return i, nil } @@ -335,11 +391,11 @@ func (m *EngineCheckpointModel) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(v.Size())) - n2, err := v.MarshalTo(dAtA[i:]) + n4, err := v.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n2 + i += n4 } } } @@ -437,6 +493,9 @@ func encodeVarintFileCheckpoints(dAtA []byte, offset int, v uint64) int { return offset + 1 } func (m *CheckpointsModel) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l if len(m.Checkpoints) > 0 { @@ -456,6 +515,9 @@ func (m *CheckpointsModel) Size() (n int) { } func (m *TableCheckpointModel) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l l = len(m.Hash) @@ -474,10 +536,20 @@ func (m *TableCheckpointModel) Size() (n int) { n += 1 + l + sovFileCheckpoints(uint64(l)) } } + if len(m.IndexEngineStatuses) > 0 { + l = 0 + for _, e := range m.IndexEngineStatuses { + l += sovFileCheckpoints(uint64(e)) + } + n += 1 + sovFileCheckpoints(uint64(l)) + l + } return n } func (m *EngineCheckpointModel) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l if m.Status != 0 { @@ -500,6 +572,9 @@ func (m *EngineCheckpointModel) Size() (n int) { } func (m *ChunkCheckpointModel) Size() (n int) { + if m == nil { + return 0 + } var l int _ = l l = len(m.Path) @@ -568,7 +643,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -596,7 +671,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -605,6 +680,9 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -625,7 +703,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -642,7 +720,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift + stringLenmapkey |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -652,6 +730,9 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthFileCheckpoints + } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -668,7 +749,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapmsglen |= (int(b) & 0x7F) << shift + mapmsglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -677,7 +758,7 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postmsgIndex := iNdEx + mapmsglen - if mapmsglen < 0 { + if postmsgIndex < 0 { return ErrInvalidLengthFileCheckpoints } if postmsgIndex > l { @@ -714,6 +795,9 @@ func (m *CheckpointsModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFileCheckpoints + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -741,7 +825,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -769,7 +853,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -778,6 +862,9 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -800,7 +887,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= (uint32(b) & 0x7F) << shift + m.Status |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -819,7 +906,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.AllocBase |= (int64(b) & 0x7F) << shift + m.AllocBase |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -838,7 +925,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -847,6 +934,9 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -855,6 +945,82 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IndexEngineStatuses = append(m.IndexEngineStatuses, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthFileCheckpoints + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.IndexEngineStatuses) == 0 { + m.IndexEngineStatuses = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.IndexEngineStatuses = append(m.IndexEngineStatuses, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field IndexEngineStatuses", wireType) + } default: iNdEx = preIndex skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) @@ -864,6 +1030,9 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFileCheckpoints + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -891,7 +1060,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -919,7 +1088,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= (uint32(b) & 0x7F) << shift + m.Status |= uint32(b&0x7F) << shift if b < 0x80 { break } @@ -938,7 +1107,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= (int(b) & 0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -947,6 +1116,9 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -967,7 +1139,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -984,7 +1156,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift + stringLenmapkey |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -994,6 +1166,9 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthFileCheckpoints + } if postStringIndexmapkey > l { return io.ErrUnexpectedEOF } @@ -1010,7 +1185,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapmsglen |= (int(b) & 0x7F) << shift + mapmsglen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -1019,7 +1194,7 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postmsgIndex := iNdEx + mapmsglen - if mapmsglen < 0 { + if postmsgIndex < 0 { return ErrInvalidLengthFileCheckpoints } if postmsgIndex > l { @@ -1056,6 +1231,9 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFileCheckpoints + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -1083,7 +1261,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - wire |= (uint64(b) & 0x7F) << shift + wire |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -1111,7 +1289,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -1121,6 +1299,9 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -1140,7 +1321,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Offset |= (int64(b) & 0x7F) << shift + m.Offset |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -1159,7 +1340,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= (int(b) & 0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } @@ -1168,6 +1349,9 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { return ErrInvalidLengthFileCheckpoints } postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } if postIndex > l { return io.ErrUnexpectedEOF } @@ -1190,7 +1374,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (int(b) & 0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } @@ -1210,7 +1394,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.EndOffset |= (int64(b) & 0x7F) << shift + m.EndOffset |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -1229,7 +1413,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Pos |= (int64(b) & 0x7F) << shift + m.Pos |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -1248,7 +1432,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.PrevRowidMax |= (int64(b) & 0x7F) << shift + m.PrevRowidMax |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -1267,7 +1451,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RowidMax |= (int64(b) & 0x7F) << shift + m.RowidMax |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -1286,7 +1470,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.KvcBytes |= (uint64(b) & 0x7F) << shift + m.KvcBytes |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -1305,7 +1489,7 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.KvcKvs |= (uint64(b) & 0x7F) << shift + m.KvcKvs |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -1329,6 +1513,9 @@ func (m *ChunkCheckpointModel) Unmarshal(dAtA []byte) error { if skippy < 0 { return ErrInvalidLengthFileCheckpoints } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthFileCheckpoints + } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } @@ -1395,10 +1582,13 @@ func skipFileCheckpoints(dAtA []byte) (n int, err error) { break } } - iNdEx += length if length < 0 { return 0, ErrInvalidLengthFileCheckpoints } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthFileCheckpoints + } return iNdEx, nil case 3: for { @@ -1427,6 +1617,9 @@ func skipFileCheckpoints(dAtA []byte) (n int, err error) { return 0, err } iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthFileCheckpoints + } } return iNdEx, nil case 4: @@ -1445,46 +1638,3 @@ var ( ErrInvalidLengthFileCheckpoints = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowFileCheckpoints = fmt.Errorf("proto: integer overflow") ) - -func init() { - proto.RegisterFile("lightning/restore/file_checkpoints.proto", fileDescriptor_file_checkpoints_168275cfec5db5bf) -} - -var fileDescriptor_file_checkpoints_168275cfec5db5bf = []byte{ - // 550 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcf, 0x6e, 0xd3, 0x3e, - 0x1c, 0x9f, 0x9b, 0x2d, 0x6d, 0xbf, 0xe9, 0xef, 0xa7, 0xca, 0x5a, 0x87, 0x55, 0xb4, 0x2a, 0x54, - 0x1c, 0x22, 0x21, 0x52, 0x18, 0x17, 0xb4, 0x63, 0xcb, 0x0e, 0x13, 0x9a, 0x40, 0x16, 0x5c, 0xb8, - 0x44, 0xf9, 0xe3, 0x26, 0x51, 0xd2, 0xb8, 0x8a, 0x93, 0x6c, 0x7d, 0x0b, 0x24, 0x1e, 0x84, 0x27, - 0xe0, 0xbe, 0x23, 0x0f, 0xc0, 0x01, 0xca, 0x8b, 0x20, 0x3b, 0x99, 0x9a, 0x4d, 0x15, 0xe2, 0xf6, - 0xfd, 0xfc, 0xf1, 0xc7, 0xce, 0xc7, 0x31, 0x58, 0x69, 0x1c, 0x46, 0x45, 0x16, 0x67, 0xe1, 0x2c, - 0x67, 0xa2, 0xe0, 0x39, 0x9b, 0x2d, 0xe3, 0x94, 0x39, 0x7e, 0xc4, 0xfc, 0x64, 0xcd, 0xe3, 0xac, - 0x10, 0xf6, 0x3a, 0xe7, 0x05, 0x1f, 0x3f, 0x0f, 0xe3, 0x22, 0x2a, 0x3d, 0xdb, 0xe7, 0xab, 0x59, - 0xc8, 0x43, 0x3e, 0x53, 0xb4, 0x57, 0x2e, 0x15, 0x52, 0x40, 0x4d, 0xb5, 0x7d, 0xfa, 0x15, 0xc1, - 0x70, 0xb1, 0x0b, 0xb9, 0xe2, 0x01, 0x4b, 0xf1, 0x1b, 0x30, 0x5a, 0xc1, 0x04, 0x99, 0x9a, 0x65, - 0x9c, 0x4d, 0xed, 0x87, 0xbe, 0x36, 0x71, 0x91, 0x15, 0xf9, 0x86, 0xb6, 0x97, 0x8d, 0x3f, 0xde, - 0x4b, 0x56, 0x06, 0x3c, 0x04, 0x2d, 0x61, 0x1b, 0x82, 0x4c, 0x64, 0xf5, 0xa9, 0x1c, 0xf1, 0x33, - 0x38, 0xaa, 0xdc, 0xb4, 0x64, 0xa4, 0x63, 0x22, 0xcb, 0x38, 0x1b, 0xd9, 0x1f, 0x5c, 0x2f, 0x65, - 0xbb, 0x85, 0x6a, 0x27, 0x5a, 0x7b, 0xce, 0x3b, 0xaf, 0xd1, 0xf4, 0x0b, 0x82, 0xe3, 0x7d, 0x1e, - 0x8c, 0xe1, 0x30, 0x72, 0x45, 0xa4, 0xc2, 0x07, 0x54, 0xcd, 0xf8, 0x04, 0x74, 0x51, 0xb8, 0x45, - 0x29, 0x88, 0x66, 0x22, 0xeb, 0x3f, 0xda, 0x20, 0x7c, 0x0a, 0xe0, 0xa6, 0x29, 0xf7, 0x1d, 0xcf, - 0x15, 0x8c, 0x1c, 0x9a, 0xc8, 0xd2, 0x68, 0x5f, 0x31, 0x73, 0x57, 0x30, 0xfc, 0x02, 0xba, 0x2c, - 0x0b, 0xe3, 0x8c, 0x09, 0xa2, 0xab, 0x8f, 0x3f, 0xb1, 0x2f, 0x14, 0x7e, 0x78, 0xae, 0x3b, 0xdb, - 0xf4, 0x1b, 0x82, 0xd1, 0x5e, 0x4b, 0xeb, 0x08, 0xe8, 0xde, 0x11, 0xce, 0x41, 0xf7, 0xa3, 0x32, - 0x4b, 0x04, 0xe9, 0x34, 0xfd, 0xee, 0x5d, 0x6f, 0x2f, 0x94, 0xa9, 0xee, 0xb7, 0x59, 0x31, 0x7e, - 0x0f, 0x46, 0x8b, 0xfe, 0x97, 0x56, 0x95, 0xfd, 0x2f, 0xad, 0xfe, 0xe8, 0xc0, 0xf1, 0x3e, 0x8f, - 0x6c, 0x75, 0xed, 0x16, 0x51, 0x13, 0xae, 0x66, 0xf9, 0x49, 0x7c, 0xb9, 0x14, 0xac, 0x50, 0xf1, - 0x1a, 0x6d, 0x10, 0x26, 0xd0, 0xf5, 0x79, 0x5a, 0xae, 0xb2, 0xba, 0xee, 0x01, 0xbd, 0x83, 0xf8, - 0x25, 0x8c, 0x44, 0xc4, 0xcb, 0x34, 0x70, 0xe2, 0xcc, 0x4f, 0xcb, 0x80, 0x39, 0x39, 0xbf, 0x76, - 0xe2, 0x40, 0x55, 0xdf, 0xa3, 0xb8, 0x16, 0x2f, 0x6b, 0x8d, 0xf2, 0xeb, 0xcb, 0x40, 0x5e, 0x11, - 0xcb, 0x02, 0xa7, 0xd9, 0xe8, 0xa8, 0xbe, 0x22, 0x96, 0x05, 0xef, 0xea, 0xbd, 0x86, 0xa0, 0xad, - 0xb9, 0xbc, 0x1e, 0xc9, 0xcb, 0x11, 0x3f, 0x85, 0xff, 0xd7, 0x39, 0xab, 0x64, 0x72, 0x1c, 0x38, - 0x2b, 0xf7, 0x86, 0x74, 0x95, 0x38, 0x90, 0x2c, 0x95, 0xe4, 0x95, 0x7b, 0x83, 0x1f, 0x43, 0x7f, - 0x67, 0xe8, 0x29, 0x43, 0x2f, 0x6f, 0x89, 0x49, 0xe5, 0x3b, 0xde, 0xa6, 0x60, 0x82, 0xf4, 0x4d, - 0x64, 0x1d, 0xd2, 0x5e, 0x52, 0xf9, 0x73, 0x89, 0xf1, 0x23, 0xe8, 0x4a, 0x31, 0xa9, 0x04, 0x01, - 0x25, 0xe9, 0x49, 0xe5, 0xbf, 0xad, 0x04, 0x7e, 0x02, 0x03, 0x29, 0xa8, 0x7f, 0x5f, 0x94, 0x2b, - 0x62, 0x98, 0xc8, 0xd2, 0xa9, 0x91, 0x54, 0xfe, 0xa2, 0xa1, 0xe6, 0xa7, 0xb7, 0xbf, 0x26, 0x07, - 0xb7, 0xdb, 0x09, 0xfa, 0xbe, 0x9d, 0xa0, 0x9f, 0xdb, 0x09, 0xfa, 0xfc, 0x7b, 0x72, 0xf0, 0xa9, - 0xdb, 0xbc, 0x65, 0x4f, 0x57, 0x8f, 0xf1, 0xd5, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x5b, 0x0d, - 0x43, 0xc6, 0xe7, 0x03, 0x00, 0x00, -} diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index c15e7689a..7909d8c7c 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -28,6 +28,7 @@ message TableCheckpointModel { uint32 status = 3; int64 alloc_base = 4; repeated EngineCheckpointModel engines = 6; + repeated uint32 index_engine_statuses = 7; } message EngineCheckpointModel { From 66ffc8afe179246c4bf19bff920a0de0d46cc407 Mon Sep 17 00:00:00 2001 From: Lonng Date: Mon, 4 Mar 2019 16:13:26 +0800 Subject: [PATCH 07/11] restore: implement checkpoint for index engine file --- lightning/config/config.go | 12 ++- lightning/restore/checkpoints.go | 51 ++++++---- lightning/restore/file_checkpoints.pb.go | 107 +++++++++++++-------- lightning/restore/file_checkpoints.proto | 1 + lightning/restore/restore.go | 115 ++++++++++++++++------- tidb-lightning.toml | 2 + 6 files changed, 195 insertions(+), 93 deletions(-) diff --git a/lightning/config/config.go b/lightning/config/config.go index 51a7c3e73..d919ee354 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -83,6 +83,7 @@ func (c *Config) String() string { type Lightning struct { common.LogConfig TableConcurrency int `toml:"table-concurrency" json:"table-concurrency"` + IndexConcurrency int `toml:"index-concurrency" json:"index-concurrency"` RegionConcurrency int `toml:"region-concurrency" json:"region-concurrency"` IOConcurrency int `toml:"io-concurrency" json:"io-concurrency"` ProfilePort int `toml:"pprof-port" json:"pprof-port"` @@ -229,11 +230,18 @@ func (cfg *Config) Load() error { } } - // If the level 1 compact configuration not found, default to true + // If the level 1 compact configuration not found, default to false if cfg.PostRestore.Level1Compact == nil { cfg.PostRestore.Level1Compact = new(bool) - *cfg.PostRestore.Level1Compact = true + *cfg.PostRestore.Level1Compact = false } + if cfg.App.TableConcurrency < 2 { + common.AppLogger.Warnf("table-concurrency should greater or equal than 2, current %v", cfg.App.TableConcurrency) + cfg.App.TableConcurrency = 2 + } + if cfg.App.IndexConcurrency < 2 { + cfg.App.IndexConcurrency = 2 + } return nil } diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index abcb43e62..a29f98941 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -41,6 +41,7 @@ const ( CheckpointStatusAllWritten CheckpointStatus = 60 CheckpointStatusClosed CheckpointStatus = 90 CheckpointStatusImported CheckpointStatus = 120 + CheckpointStatusIndexImported CheckpointStatus = 140 CheckpointStatusAlteredAutoInc CheckpointStatus = 150 CheckpointStatusChecksumSkipped CheckpointStatus = 170 CheckpointStatusChecksummed CheckpointStatus = 180 @@ -108,8 +109,9 @@ type ChunkCheckpoint struct { } type EngineCheckpoint struct { - Status CheckpointStatus - Chunks []*ChunkCheckpoint // a sorted array + EngineID int + Status CheckpointStatus + Chunks []*ChunkCheckpoint // a sorted array } type TableCheckpoint struct { @@ -192,7 +194,7 @@ func (merger *StatusCheckpointMerger) SetInvalid() { } func (merger *StatusCheckpointMerger) MergeInto(cpd *TableCheckpointDiff) { - if merger.EngineID == -1 || merger.Status <= CheckpointStatusMaxInvalid { + if merger.EngineID == invalidEngineID || merger.Status <= CheckpointStatusMaxInvalid { cpd.status = merger.Status cpd.hasStatus = true } @@ -432,10 +434,18 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab if err := engineRows.Scan(&engineID, &status); err != nil { return errors.Trace(err) } - for len(cp.Engines) <= engineID { - cp.Engines = append(cp.Engines, new(EngineCheckpoint)) + var found bool + for _, engine := range cp.Engines { + if engineID == engine.EngineID { + engine.Status = CheckpointStatus(status) + found = true + break + } + } + if !found { + checkpoint := &EngineCheckpoint{EngineID: engineID, Status: CheckpointStatus(status)} + cp.Engines = append(cp.Engines, checkpoint) } - cp.Engines[engineID].Status = CheckpointStatus(status) } if err := engineRows.Err(); err != nil { return errors.Trace(err) @@ -472,6 +482,8 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab return errors.Trace(err) } value.Checksum = verify.MakeKVChecksum(kvcBytes, kvcKVs, kvcChecksum) + // It is ok to use engineID here, because index engine file will not + // contains any chunks cp.Engines[engineID].Chunks = append(cp.Engines[engineID].Chunks, value) } if err := chunkRows.Err(); err != nil { @@ -527,14 +539,14 @@ func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tab } defer chunkStmt.Close() - for engineID, engine := range checkpoints { - _, err = engineStmt.ExecContext(c, tableName, engineID, engine.Status) + for _, engine := range checkpoints { + _, err = engineStmt.ExecContext(c, tableName, engine.EngineID, engine.Status) if err != nil { return errors.Trace(err) } for _, value := range engine.Chunks { _, err = chunkStmt.ExecContext( - c, tableName, engineID, + c, tableName, engine.EngineID, value.Key.Path, value.Key.Offset, value.Columns, value.ShouldIncludeRowID, value.Chunk.Offset, value.Chunk.EndOffset, value.Chunk.PrevRowIDMax, value.Chunk.RowIDMax, value.Checksum.SumSize(), value.Checksum.SumKVS(), value.Checksum.Sum(), @@ -700,8 +712,9 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC for _, engineModel := range tableModel.Engines { engine := &EngineCheckpoint{ - Status: CheckpointStatus(engineModel.Status), - Chunks: make([]*ChunkCheckpoint, 0, len(engineModel.Chunks)), + EngineID: int(engineModel.EngineId), + Status: CheckpointStatus(engineModel.Status), + Chunks: make([]*ChunkCheckpoint, 0, len(engineModel.Chunks)), } for _, chunkModel := range engineModel.Chunks { @@ -737,15 +750,12 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN defer cpdb.lock.Unlock() tableModel := cpdb.checkpoints.Checkpoints[tableName] - for len(tableModel.Engines) < len(checkpoints) { - tableModel.Engines = append(tableModel.Engines, &EngineCheckpointModel{ - Status: uint32(CheckpointStatusLoaded), - Chunks: make(map[string]*ChunkCheckpointModel), - }) - } - - for engineID, engine := range checkpoints { - engineModel := tableModel.Engines[engineID] + for _, engine := range checkpoints { + engineModel := &EngineCheckpointModel{ + EngineId: int32(engine.EngineID), + Status: uint32(CheckpointStatusLoaded), + Chunks: make(map[string]*ChunkCheckpointModel), + } for _, value := range engine.Chunks { key := value.Key.String() chunk, ok := engineModel.Chunks[key] @@ -766,6 +776,7 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk.KvcKvs = value.Checksum.SumKVS() chunk.KvcChecksum = value.Checksum.Sum() } + tableModel.Engines = append(tableModel.Engines, engineModel) } return errors.Trace(cpdb.save()) diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index cf0ab9c0b..82659a77a 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -105,7 +105,8 @@ var xxx_messageInfo_TableCheckpointModel proto.InternalMessageInfo type EngineCheckpointModel struct { Status uint32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // key is "$path:$offset" - Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + EngineId int32 `protobuf:"varint,3,opt,name=engine_id,json=engineId,proto3" json:"engine_id,omitempty"` } func (m *EngineCheckpointModel) Reset() { *m = EngineCheckpointModel{} } @@ -202,44 +203,45 @@ func init() { } var fileDescriptor_c47ec4f2f281cd62 = []byte{ - // 583 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcb, 0x6e, 0xd3, 0x4c, - 0x14, 0xce, 0xc4, 0x6d, 0x2e, 0x27, 0xe9, 0xaf, 0x6a, 0xfe, 0xa6, 0x8c, 0x8a, 0xb0, 0x4c, 0x04, - 0x92, 0x25, 0x84, 0x03, 0x65, 0x83, 0xba, 0x6c, 0xe9, 0xa2, 0x42, 0x15, 0x68, 0x80, 0x0d, 0x1b, - 0xcb, 0x97, 0x89, 0x6d, 0xd9, 0x99, 0x89, 0x3c, 0xb6, 0xdb, 0xbe, 0x05, 0x6f, 0xc2, 0x13, 0xb0, - 0xa6, 0xcb, 0x2e, 0x59, 0xb0, 0x80, 0xf6, 0x45, 0xd0, 0x8c, 0x5d, 0xc5, 0xad, 0x22, 0xc4, 0xee, - 0x7c, 0x97, 0xf9, 0xce, 0x58, 0x9f, 0xc6, 0x60, 0x67, 0x49, 0x14, 0x17, 0x3c, 0xe1, 0xd1, 0x2c, - 0x67, 0xb2, 0x10, 0x39, 0x9b, 0xcd, 0x93, 0x8c, 0xb9, 0x41, 0xcc, 0x82, 0x74, 0x29, 0x12, 0x5e, - 0x48, 0x67, 0x99, 0x8b, 0x42, 0xec, 0x3d, 0x8f, 0x92, 0x22, 0x2e, 0x7d, 0x27, 0x10, 0x8b, 0x59, - 0x24, 0x22, 0x31, 0xd3, 0xb4, 0x5f, 0xce, 0x35, 0xd2, 0x40, 0x4f, 0xb5, 0x7d, 0xfa, 0x15, 0xc1, - 0xf6, 0xd1, 0x2a, 0xe4, 0x54, 0x84, 0x2c, 0xc3, 0x6f, 0x60, 0xd4, 0x0a, 0x26, 0xc8, 0x32, 0xec, - 0xd1, 0xfe, 0xd4, 0xb9, 0xef, 0x6b, 0x13, 0xc7, 0xbc, 0xc8, 0x2f, 0x68, 0xfb, 0xd8, 0xde, 0xa7, - 0x3b, 0xc9, 0xda, 0x80, 0xb7, 0xc1, 0x48, 0xd9, 0x05, 0x41, 0x16, 0xb2, 0x87, 0x54, 0x8d, 0xf8, - 0x19, 0x6c, 0x56, 0x5e, 0x56, 0x32, 0xd2, 0xb5, 0x90, 0x3d, 0xda, 0x9f, 0x38, 0x1f, 0x3d, 0x3f, - 0x63, 0xab, 0x83, 0x7a, 0x13, 0xad, 0x3d, 0x07, 0xdd, 0xd7, 0x68, 0xfa, 0x1d, 0xc1, 0xce, 0x3a, - 0x0f, 0xc6, 0xb0, 0x11, 0x7b, 0x32, 0xd6, 0xe1, 0x63, 0xaa, 0x67, 0xbc, 0x0b, 0x3d, 0x59, 0x78, - 0x45, 0x29, 0x89, 0x61, 0x21, 0x7b, 0x8b, 0x36, 0x08, 0x3f, 0x02, 0xf0, 0xb2, 0x4c, 0x04, 0xae, - 0xef, 0x49, 0x46, 0x36, 0x2c, 0x64, 0x1b, 0x74, 0xa8, 0x99, 0x43, 0x4f, 0x32, 0xfc, 0x02, 0xfa, - 0x8c, 0x47, 0x09, 0x67, 0x92, 0xf4, 0xf4, 0xc7, 0xef, 0x3a, 0xc7, 0x1a, 0xdf, 0xbf, 0xd7, 0xad, - 0x0d, 0xef, 0xc3, 0x24, 0xe1, 0x21, 0x3b, 0x77, 0x6b, 0xc2, 0xad, 0xf7, 0x30, 0x49, 0xfa, 0x96, - 0x61, 0x6f, 0xd1, 0xff, 0xb5, 0x58, 0x67, 0x7c, 0x68, 0xa4, 0xe9, 0x37, 0x04, 0x93, 0xb5, 0xb1, - 0xad, 0x6b, 0xa3, 0x3b, 0xd7, 0x3e, 0x80, 0x5e, 0x10, 0x97, 0x3c, 0x95, 0xa4, 0xdb, 0x74, 0xb2, - 0xf6, 0xbc, 0x73, 0xa4, 0x4d, 0x75, 0x27, 0xcd, 0x89, 0xbd, 0xf7, 0x30, 0x6a, 0xd1, 0xff, 0xd2, - 0x84, 0xb6, 0xff, 0xa5, 0x89, 0x9f, 0x5d, 0xd8, 0x59, 0xe7, 0x51, 0x4d, 0x2c, 0xbd, 0x22, 0x6e, - 0xc2, 0xf5, 0xac, 0x3e, 0x49, 0xcc, 0xe7, 0x92, 0x15, 0x3a, 0xde, 0xa0, 0x0d, 0xc2, 0x04, 0xfa, - 0x81, 0xc8, 0xca, 0x05, 0xaf, 0x2b, 0x1a, 0xd3, 0x5b, 0x88, 0x5f, 0xc2, 0x44, 0xc6, 0xa2, 0xcc, - 0x42, 0x37, 0xe1, 0x41, 0x56, 0x86, 0xcc, 0xcd, 0xc5, 0x99, 0x9b, 0x84, 0xba, 0xae, 0x01, 0xc5, - 0xb5, 0x78, 0x52, 0x6b, 0x54, 0x9c, 0x9d, 0x84, 0xaa, 0x56, 0xc6, 0x43, 0xb7, 0x59, 0xb4, 0x59, - 0xd7, 0xca, 0x78, 0xf8, 0xae, 0xde, 0xb5, 0x0d, 0xc6, 0x52, 0xa8, 0x4a, 0x15, 0xaf, 0x46, 0xfc, - 0x04, 0xfe, 0x5b, 0xe6, 0xac, 0x52, 0xc9, 0x49, 0xe8, 0x2e, 0xbc, 0x73, 0xd2, 0xd7, 0xe2, 0x58, - 0xb1, 0x54, 0x91, 0xa7, 0xde, 0x39, 0x7e, 0x08, 0xc3, 0x95, 0x61, 0xa0, 0x0d, 0x83, 0xbc, 0x25, - 0xa6, 0x55, 0xe0, 0xfa, 0x17, 0x05, 0x93, 0x64, 0x68, 0x21, 0x7b, 0x83, 0x0e, 0xd2, 0x2a, 0x38, - 0x54, 0x18, 0x3f, 0x80, 0xbe, 0x12, 0xd3, 0x4a, 0x12, 0xd0, 0x52, 0x2f, 0xad, 0x82, 0xb7, 0x95, - 0xc4, 0x8f, 0x61, 0xac, 0x04, 0xfd, 0x5e, 0x64, 0xb9, 0x20, 0x23, 0x0b, 0xd9, 0x3d, 0x3a, 0x4a, - 0xab, 0xe0, 0xa8, 0xa1, 0x0e, 0x9f, 0x5e, 0xfe, 0x36, 0x3b, 0x97, 0xd7, 0x26, 0xba, 0xba, 0x36, - 0xd1, 0xaf, 0x6b, 0x13, 0x7d, 0xb9, 0x31, 0x3b, 0x57, 0x37, 0x66, 0xe7, 0xc7, 0x8d, 0xd9, 0xf9, - 0xdc, 0x6f, 0xfe, 0x03, 0x7e, 0x4f, 0x3f, 0xe4, 0x57, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, - 0xf1, 0xe2, 0x3d, 0x23, 0x04, 0x00, 0x00, + // 597 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xcd, 0xc4, 0x6d, 0x7e, 0x6e, 0xd2, 0x4f, 0xd5, 0x7c, 0x4d, 0x19, 0x15, 0x61, 0x99, 0x08, + 0x24, 0x4b, 0x88, 0x04, 0xca, 0x06, 0x75, 0xd9, 0xd2, 0x45, 0x85, 0x2a, 0xd0, 0x00, 0x1b, 0x36, + 0x96, 0x63, 0x4f, 0x6c, 0xcb, 0xce, 0x4c, 0xe4, 0xb1, 0xdd, 0xf6, 0x2d, 0x78, 0x13, 0x1e, 0x83, + 0x2e, 0xbb, 0x44, 0x88, 0x05, 0xb4, 0x2f, 0x82, 0x66, 0xc6, 0x55, 0xdc, 0x2a, 0x42, 0xec, 0xee, + 0x3d, 0xe7, 0xdc, 0x33, 0x77, 0x7c, 0x34, 0x06, 0x37, 0x4b, 0xa2, 0xb8, 0xe0, 0x09, 0x8f, 0xa6, + 0x39, 0x93, 0x85, 0xc8, 0xd9, 0x74, 0x9e, 0x64, 0xcc, 0x0b, 0x62, 0x16, 0xa4, 0x4b, 0x91, 0xf0, + 0x42, 0x4e, 0x96, 0xb9, 0x28, 0xc4, 0xde, 0xf3, 0x28, 0x29, 0xe2, 0x72, 0x36, 0x09, 0xc4, 0x62, + 0x1a, 0x89, 0x48, 0x4c, 0x35, 0x3c, 0x2b, 0xe7, 0xba, 0xd3, 0x8d, 0xae, 0x8c, 0x7c, 0xfc, 0x15, + 0xc1, 0xf6, 0xd1, 0xca, 0xe4, 0x54, 0x84, 0x2c, 0xc3, 0x6f, 0x60, 0xd0, 0x30, 0x26, 0xc8, 0xb1, + 0xdc, 0xc1, 0xfe, 0x78, 0x72, 0x5f, 0xd7, 0x04, 0x8e, 0x79, 0x91, 0x5f, 0xd0, 0xe6, 0xd8, 0xde, + 0xa7, 0x3b, 0xce, 0x5a, 0x80, 0xb7, 0xc1, 0x4a, 0xd9, 0x05, 0x41, 0x0e, 0x72, 0xfb, 0x54, 0x95, + 0xf8, 0x19, 0x6c, 0x56, 0x7e, 0x56, 0x32, 0xd2, 0x76, 0x90, 0x3b, 0xd8, 0x1f, 0x4d, 0x3e, 0xfa, + 0xb3, 0x8c, 0xad, 0x06, 0xf5, 0x49, 0xd4, 0x68, 0x0e, 0xda, 0xaf, 0xd1, 0xf8, 0x1b, 0x82, 0x9d, + 0x75, 0x1a, 0x8c, 0x61, 0x23, 0xf6, 0x65, 0xac, 0xcd, 0x87, 0x54, 0xd7, 0x78, 0x17, 0x3a, 0xb2, + 0xf0, 0x8b, 0x52, 0x12, 0xcb, 0x41, 0xee, 0x16, 0xad, 0x3b, 0xfc, 0x08, 0xc0, 0xcf, 0x32, 0x11, + 0x78, 0x33, 0x5f, 0x32, 0xb2, 0xe1, 0x20, 0xd7, 0xa2, 0x7d, 0x8d, 0x1c, 0xfa, 0x92, 0xe1, 0x17, + 0xd0, 0x65, 0x3c, 0x4a, 0x38, 0x93, 0xa4, 0xa3, 0x2f, 0xbf, 0x3b, 0x39, 0xd6, 0xfd, 0xfd, 0xbd, + 0x6e, 0x65, 0x78, 0x1f, 0x46, 0x09, 0x0f, 0xd9, 0xb9, 0x67, 0x00, 0xcf, 0x9c, 0xc3, 0x24, 0xe9, + 0x3a, 0x96, 0xbb, 0x45, 0xff, 0xd7, 0xa4, 0xf1, 0xf8, 0x50, 0x53, 0xe3, 0x1f, 0x08, 0x46, 0x6b, + 0x6d, 0x1b, 0x6b, 0xa3, 0x3b, 0x6b, 0x1f, 0x40, 0x27, 0x88, 0x4b, 0x9e, 0x4a, 0xd2, 0xae, 0x33, + 0x59, 0x3b, 0x3f, 0x39, 0xd2, 0x22, 0x93, 0x49, 0x3d, 0x81, 0x1f, 0x42, 0xbf, 0xde, 0x2d, 0x09, + 0xf5, 0xd7, 0xd8, 0xa4, 0x3d, 0x03, 0x9c, 0x84, 0x7b, 0xef, 0x61, 0xd0, 0x98, 0xf9, 0x97, 0x98, + 0xb4, 0xfc, 0x2f, 0x31, 0xfd, 0x6c, 0xc3, 0xce, 0x3a, 0x8d, 0x8a, 0x69, 0xe9, 0x17, 0x71, 0x6d, + 0xae, 0x6b, 0x75, 0x5f, 0x31, 0x9f, 0x4b, 0x56, 0x68, 0x7b, 0x8b, 0xd6, 0x1d, 0x26, 0xd0, 0x0d, + 0x44, 0x56, 0x2e, 0xb8, 0xc9, 0x6f, 0x48, 0x6f, 0x5b, 0xfc, 0x12, 0x46, 0x32, 0x16, 0x65, 0x16, + 0x7a, 0x09, 0x0f, 0xb2, 0x32, 0x64, 0x5e, 0x2e, 0xce, 0xd4, 0xcd, 0x54, 0x96, 0x3d, 0x8a, 0x0d, + 0x79, 0x62, 0x38, 0x2a, 0xce, 0x4e, 0x42, 0x95, 0x39, 0xe3, 0xa1, 0x57, 0x1f, 0xb4, 0x69, 0x32, + 0x67, 0x3c, 0x7c, 0x67, 0xce, 0xda, 0x06, 0x6b, 0x29, 0x54, 0xde, 0x0a, 0x57, 0x25, 0x7e, 0x02, + 0xff, 0x2d, 0x73, 0x56, 0x29, 0xe7, 0x24, 0xf4, 0x16, 0xfe, 0x39, 0xe9, 0x6a, 0x72, 0xa8, 0x50, + 0xaa, 0xc0, 0x53, 0xff, 0x5c, 0x7d, 0xd7, 0x95, 0xa0, 0xa7, 0x05, 0xbd, 0xbc, 0x41, 0xa6, 0x55, + 0xe0, 0xcd, 0x2e, 0x0a, 0x26, 0x49, 0xdf, 0x41, 0xee, 0x06, 0xed, 0xa5, 0x55, 0x70, 0xa8, 0x7a, + 0xfc, 0x00, 0xba, 0x8a, 0x4c, 0x2b, 0x49, 0x40, 0x53, 0x9d, 0xb4, 0x0a, 0xde, 0x56, 0x12, 0x3f, + 0x86, 0xa1, 0x22, 0xf4, 0x63, 0x92, 0xe5, 0x82, 0x0c, 0x1c, 0xe4, 0x76, 0xe8, 0x20, 0xad, 0x82, + 0xa3, 0x1a, 0x3a, 0x7c, 0x7a, 0xf9, 0xdb, 0x6e, 0x5d, 0x5e, 0xdb, 0xe8, 0xea, 0xda, 0x46, 0xbf, + 0xae, 0x6d, 0xf4, 0xe5, 0xc6, 0x6e, 0x5d, 0xdd, 0xd8, 0xad, 0xef, 0x37, 0x76, 0xeb, 0x73, 0xb7, + 0xfe, 0x49, 0xcc, 0x3a, 0xfa, 0x95, 0xbf, 0xfa, 0x13, 0x00, 0x00, 0xff, 0xff, 0x74, 0x46, 0x29, + 0xb6, 0x40, 0x04, 0x00, 0x00, } func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { @@ -399,6 +401,11 @@ func (m *EngineCheckpointModel) MarshalTo(dAtA []byte) (int, error) { } } } + if m.EngineId != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintFileCheckpoints(dAtA, i, uint64(m.EngineId)) + } return i, nil } @@ -568,6 +575,9 @@ func (m *EngineCheckpointModel) Size() (n int) { n += mapEntrySize + 1 + sovFileCheckpoints(uint64(mapEntrySize)) } } + if m.EngineId != 0 { + n += 1 + sovFileCheckpoints(uint64(m.EngineId)) + } return n } @@ -1222,6 +1232,25 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } m.Chunks[mapkey] = mapvalue iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EngineId", wireType) + } + m.EngineId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EngineId |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index 7909d8c7c..6ae031312 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -35,6 +35,7 @@ message EngineCheckpointModel { uint32 status = 1; // key is "$path:$offset" map chunks = 2; + int32 engine_id = 3; } message ChunkCheckpointModel { diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 43da4c299..d31e14cd1 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" "io" + "math" "net/http" "os" "regexp" @@ -29,9 +30,13 @@ import ( "github.com/coreos/go-semver/semver" "github.com/cznic/mathutil" + "github.com/pingcap/errors" sstpb "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/parser/model" + tidbcfg "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/kvencoder" "github.com/pingcap/tidb-lightning/lightning/common" "github.com/pingcap/tidb-lightning/lightning/config" @@ -40,11 +45,6 @@ import ( "github.com/pingcap/tidb-lightning/lightning/mydump" verify "github.com/pingcap/tidb-lightning/lightning/verification" "github.com/pingcap/tidb-lightning/lightning/worker" - - "github.com/pingcap/errors" - tidbcfg "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/meta/autoid" - "github.com/pingcap/tidb/util/kvencoder" ) const ( @@ -56,6 +56,11 @@ const ( defaultGCLifeTime = 100 * time.Hour ) +const ( + indexEngineID = -1 + invalidEngineID = math.MinInt64 +) + const ( compactStateIdle int32 = iota compactStateDoing @@ -112,6 +117,7 @@ type RestoreController struct { dbMetas []*mydump.MDDatabaseMeta dbInfos map[string]*TidbDBInfo tableWorkers *worker.Pool + indexWorkers *worker.Pool regionWorkers *worker.Pool ioWorkers *worker.Pool importer *kv.Importer @@ -149,6 +155,7 @@ func NewRestoreController(ctx context.Context, dbMetas []*mydump.MDDatabaseMeta, cfg: cfg, dbMetas: dbMetas, tableWorkers: worker.NewPool(ctx, cfg.App.TableConcurrency, "table"), + indexWorkers: worker.NewPool(ctx, cfg.App.IndexConcurrency, "index"), regionWorkers: worker.NewPool(ctx, cfg.App.RegionConcurrency, "region"), ioWorkers: worker.NewPool(ctx, cfg.App.IOConcurrency, "io"), importer: importer, @@ -503,20 +510,44 @@ func (t *TableRestore) restoreTable( } // 2. Restore engines (if still needed) - indexWorker := rc.tableWorkers.Apply() - defer rc.tableWorkers.Recycle(indexWorker) - indexEngine, err := rc.importer.OpenEngine(ctx, t.tableName, -1) - if err != nil { - return errors.Trace(err) + var indexEngineCp *EngineCheckpoint + for _, checkpoint := range cp.Engines { + if checkpoint.EngineID == indexEngineID { + indexEngineCp = checkpoint + break + } + } + if indexEngineCp == nil { + return fmt.Errorf("table %v index engine checkpoint not found", t.tableName) + } + var indexEngine *kv.OpenedEngine + var closedIndexEngine *kv.ClosedEngine + if indexEngineCp.Status < CheckpointStatusImported { + indexWorker := rc.indexWorkers.Apply() + defer rc.indexWorkers.Recycle(indexWorker) + var err error + indexEngine, err = rc.importer.OpenEngine(ctx, t.tableName, indexEngineID) + if err != nil { + return errors.Trace(err) + } } - if cp.Status < CheckpointStatusImported { - timer := time.Now() + // The table checkpoint status set to `CheckpointStatusIndexImported` only if + // both all data engines and the index engine had been imported to TiKV. + if cp.Status < CheckpointStatusIndexImported { + // The table checkpoint status less than `CheckpointStatusIndexImported` implies + // that index engine checkpoint status less than `CheckpointStatusImported`. + // So the index engine must be found in above process + if indexEngine == nil { + return fmt.Errorf("table checkpoint status %v incompitable with index engine checkpoint status %v", + cp.Status, indexEngineCp.Status) + } + timer := time.Now() var wg sync.WaitGroup var engineErr common.OnceError - for engineID, engine := range cp.Engines { + for _, engine := range cp.Engines { select { case <-ctx.Done(): return ctx.Err() @@ -526,6 +557,11 @@ func (t *TableRestore) restoreTable( break } + // Should skip index engine + if engine.EngineID < 0 { + continue + } + wg.Add(1) // Note: We still need tableWorkers to control the concurrency of tables. @@ -548,25 +584,32 @@ func (t *TableRestore) restoreTable( if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } - }(restoreWorker, engineID, engine) + }(restoreWorker, engine.EngineID, engine) } wg.Wait() common.AppLogger.Infof("[%s] import whole table takes %v", t.tableName, time.Since(timer)) err := engineErr.Get() - rc.saveStatusCheckpoint(t.tableName, -1, err, CheckpointStatusImported) if err != nil { return errors.Trace(err) } + + // If index engine file has been closed but not imported only if context cancel occurred + // when `importKV()` execution, so `UnsafeCloseEngine` and continue import it. + if indexEngineCp.Status == CheckpointStatusClosed { + closedIndexEngine, err = rc.importer.UnsafeCloseEngine(ctx, t.tableName, indexEngineID) + } else { + closedIndexEngine, err = indexEngine.Close(ctx) + rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusClosed) + } + if err != nil { + common.AppLogger.Errorf("[kv-deliver] index engine closed error: %s", errors.ErrorStack(err)) + return errors.Trace(err) + } } // 3. Post-process - closedIndexEngine, err := indexEngine.Close(ctx) - if err != nil { - common.AppLogger.Errorf("[kv-deliver] index engine closed error: %s", errors.ErrorStack(err)) - return errors.Trace(err) - } return errors.Trace(t.postProcess(ctx, rc, cp, closedIndexEngine)) } @@ -718,13 +761,17 @@ func (t *TableRestore) importEngine( } func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint, indexEngine *kv.ClosedEngine) error { - // the lock ensures the import() step will not be concurrent. - rc.postProcessLock.Lock() - err := t.importKV(ctx, indexEngine) - rc.postProcessLock.Unlock() - if err != nil { - common.AppLogger.Errorf("[%[1]s] failed to import index engine: %v", t.tableName, err.Error()) - return errors.Trace(err) + if cp.Status < CheckpointStatusIndexImported { + // the lock ensures the import() step will not be concurrent. + rc.postProcessLock.Lock() + err := t.importKV(ctx, indexEngine) + rc.postProcessLock.Unlock() + rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusIndexImported) + if err != nil { + common.AppLogger.Errorf("[%[1]s] failed to import index engine: %v", t.tableName, err.Error()) + return errors.Trace(err) + } } setSessionConcurrencyVars(ctx, rc.tidbMgr.db, rc.cfg.TiDB) @@ -734,7 +781,7 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c rc.alterTableLock.Lock() err := t.restoreTableMeta(ctx, rc.tidbMgr.db) rc.alterTableLock.Unlock() - rc.saveStatusCheckpoint(t.tableName, -1, err, CheckpointStatusAlteredAutoInc) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusAlteredAutoInc) if err != nil { common.AppLogger.Errorf( "[%[1]s] failed to AUTO TABLE %[1]s SET AUTO_INCREMENT=%[2]d : %[3]v", @@ -748,10 +795,10 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c if cp.Status < CheckpointStatusChecksummed { if !rc.cfg.PostRestore.Checksum { common.AppLogger.Infof("[%s] Skip checksum.", t.tableName) - rc.saveStatusCheckpoint(t.tableName, -1, nil, CheckpointStatusChecksumSkipped) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, nil, CheckpointStatusChecksumSkipped) } else { err := t.compareChecksum(ctx, rc.tidbMgr.db, cp) - rc.saveStatusCheckpoint(t.tableName, -1, err, CheckpointStatusChecksummed) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusChecksummed) if err != nil { common.AppLogger.Errorf("[%s] checksum failed: %v", t.tableName, err.Error()) return errors.Trace(err) @@ -763,10 +810,10 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c if cp.Status < CheckpointStatusAnalyzed { if !rc.cfg.PostRestore.Analyze { common.AppLogger.Infof("[%s] Skip analyze.", t.tableName) - rc.saveStatusCheckpoint(t.tableName, -1, nil, CheckpointStatusAnalyzeSkipped) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, nil, CheckpointStatusAnalyzeSkipped) } else { err := t.analyzeTable(ctx, rc.tidbMgr.db) - rc.saveStatusCheckpoint(t.tableName, -1, err, CheckpointStatusAnalyzed) + rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusAnalyzed) if err != nil { common.AppLogger.Errorf("[%s] analyze failed: %v", t.tableName, err.Error()) return errors.Trace(err) @@ -1026,6 +1073,7 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e for chunk.EngineID >= len(cp.Engines) { cp.Engines = append(cp.Engines, &EngineCheckpoint{Status: CheckpointStatusLoaded}) } + cp.Engines[chunk.EngineID].EngineID = chunk.EngineID cp.Engines[chunk.EngineID].Chunks = append(cp.Engines[chunk.EngineID].Chunks, &ChunkCheckpoint{ Key: ChunkCheckpointKey{ Path: chunk.File, @@ -1036,6 +1084,9 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e }) } + // Add index engine checkpoint + cp.Engines = append(cp.Engines, &EngineCheckpoint{EngineID: indexEngineID, Status: CheckpointStatusLoaded}) + common.AppLogger.Infof("[%s] load %d engines and %d chunks takes %v", t.tableName, len(cp.Engines), len(chunks), time.Since(timer)) return nil } diff --git a/tidb-lightning.toml b/tidb-lightning.toml index d8d965378..8cd9cadd9 100644 --- a/tidb-lightning.toml +++ b/tidb-lightning.toml @@ -7,6 +7,8 @@ pprof-port = 8289 # check if the cluster satisfies the minimum requirement before starting # check-requirements = true +# index-concurrency controls the maximum handled index concurrently while reading Mydumper SQL files. It can affect the tikv-importer disk usage. +index-concurrency = 2 # table-concurrency controls the maximum handled tables concurrently while reading Mydumper SQL files. It can affect the tikv-importer memory usage. table-concurrency = 8 # region-concurrency changes the concurrency number of data. It is set to the number of logical CPU cores by default and needs no configuration. From b266c796b7299e897a4bba1e33d866965678bbeb Mon Sep 17 00:00:00 2001 From: Lonng Date: Tue, 5 Mar 2019 11:30:33 +0800 Subject: [PATCH 08/11] tests: add failpoint for CheckpointStatusIndexImported --- lightning/config/config.go | 1 - lightning/restore/restore.go | 50 ++++++++++++++++++++++++--------- tests/checkpoint/run.sh | 2 +- tests/checkpoint_engines/run.sh | 20 ++++++++----- 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/lightning/config/config.go b/lightning/config/config.go index d919ee354..4ffc3b2c9 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -237,7 +237,6 @@ func (cfg *Config) Load() error { } if cfg.App.TableConcurrency < 2 { - common.AppLogger.Warnf("table-concurrency should greater or equal than 2, current %v", cfg.App.TableConcurrency) cfg.App.TableConcurrency = 2 } if cfg.App.IndexConcurrency < 2 { diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index d31e14cd1..e3107963b 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -348,19 +348,42 @@ func (rc *RestoreController) listenCheckpointUpdates(wg *sync.WaitGroup) { lock.Unlock() + // Note: about gofailLabel1, gofailLabel2, etc.., just a hack way + // for avoiding generated error message e.g: + // `failpoint: "github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported" got value 140 of type "int" but expected type "int"` + // gofail: var FailIfImportedChunk struct{} // if _, ok := scp.merger.(*ChunkCheckpointMerger); ok { // wg.Wait() // panic("forcing failure due to FailIfImportedChunk") // } - // continue + // goto gofailLabel1 + gofailLabel1: + if false { + goto gofailLabel1 + } // gofail: var FailIfStatusBecomes int // if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == FailIfStatusBecomes { // wg.Wait() // panic("forcing failure due to FailIfStatusBecomes") // } - // continue + // goto gofailLabel2 + gofailLabel2: + if false { + goto gofailLabel2 + } + + // gofail: var FailIfIndexEngineImported int + // if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID == invalidEngineID && merger.Status == CheckpointStatusIndexImported && FailIfIndexEngineImported > 0 { + // wg.Wait() + // panic("forcing failure due to FailIfIndexEngineImported") + // } + // goto gofailLabel3 + gofailLabel3: + if false { + goto gofailLabel3 + } } } @@ -520,21 +543,22 @@ func (t *TableRestore) restoreTable( if indexEngineCp == nil { return fmt.Errorf("table %v index engine checkpoint not found", t.tableName) } - var indexEngine *kv.OpenedEngine - var closedIndexEngine *kv.ClosedEngine - if indexEngineCp.Status < CheckpointStatusImported { - indexWorker := rc.indexWorkers.Apply() - defer rc.indexWorkers.Recycle(indexWorker) - var err error - indexEngine, err = rc.importer.OpenEngine(ctx, t.tableName, indexEngineID) - if err != nil { - return errors.Trace(err) - } - } // The table checkpoint status set to `CheckpointStatusIndexImported` only if // both all data engines and the index engine had been imported to TiKV. + var indexEngine *kv.OpenedEngine + var closedIndexEngine *kv.ClosedEngine if cp.Status < CheckpointStatusIndexImported { + if indexEngineCp.Status < CheckpointStatusImported { + indexWorker := rc.indexWorkers.Apply() + defer rc.indexWorkers.Recycle(indexWorker) + var err error + indexEngine, err = rc.importer.OpenEngine(ctx, t.tableName, indexEngineID) + if err != nil { + return errors.Trace(err) + } + } + // The table checkpoint status less than `CheckpointStatusIndexImported` implies // that index engine checkpoint status less than `CheckpointStatusImported`. // So the index engine must be found in above process diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index cc24a0146..c77d007e7 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -57,7 +57,7 @@ PARTIAL_IMPORT_QUERY="$PARTIAL_IMPORT_QUERY AS s;" # Set the failpoint to kill the lightning instance as soon as one table is imported # If checkpoint does work, this should only kill 9 instances of lightnings. -export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120)' +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)' # Start importing the tables. run_sql 'DROP DATABASE IF EXISTS cppk_tsr' diff --git a/tests/checkpoint_engines/run.sh b/tests/checkpoint_engines/run.sh index 605f1aab3..672d779bd 100755 --- a/tests/checkpoint_engines/run.sh +++ b/tests/checkpoint_engines/run.sh @@ -18,14 +18,18 @@ set -eu # First, verify that a normal operation is fine. rm -f "$TEST_DIR/lightning-checkpoint-engines.log" +rm -f "/tmp/tidb_lightning_checkpoint.pb" run_sql 'DROP DATABASE IF EXISTS cpeng;' run_lightning -# Check that we have indeed opened 4 engines +# Check that we have indeed opened 6 engines (index + data engine) +DATA_ENGINE_COUNT=4 +INDEX_ENGINE_COUNT=2 +ENGINE_COUNT=6 OPEN_ENGINES_COUNT=$(grep 'open engine' "$TEST_DIR/lightning-checkpoint-engines.log" | wc -l) echo "Number of open engines: $OPEN_ENGINES_COUNT" -[ "$OPEN_ENGINES_COUNT" -eq 4 ] +[ "$OPEN_ENGINES_COUNT" -eq $ENGINE_COUNT ] # Check that everything is correctly imported run_sql 'SELECT count(*), sum(c) FROM cpeng.a' @@ -39,11 +43,13 @@ check_contains 'sum(c): 46' # Now, verify it works with checkpoints as well. run_sql 'DROP DATABASE cpeng;' +rm -f "/tmp/tidb_lightning_checkpoint.pb" -export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120)' +# Data engine part +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfStatusBecomes=return(120);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(140)' set +e -for i in $(seq "$OPEN_ENGINES_COUNT"); do - echo "******** Importing Table Now (step $i/4) ********" +for i in $(seq "$ENGINE_COUNT"); do + echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" run_lightning 2> /dev/null [ $? -ne 0 ] || exit 1 done @@ -65,8 +71,8 @@ check_contains 'sum(c): 46' run_sql 'DROP DATABASE cpeng;' set +e -for i in $(seq "$OPEN_ENGINES_COUNT"); do - echo "******** Importing Table Now (step $i/4) ********" +for i in $(seq "$ENGINE_COUNT"); do + echo "******** Importing Table Now (step $i/$ENGINE_COUNT) ********" run_lightning mysql 2> /dev/null [ $? -ne 0 ] || exit 1 done From dce6be8692dcc7f79072cce994afe7f1c51d9f1f Mon Sep 17 00:00:00 2001 From: Lonng Date: Wed, 6 Mar 2019 15:44:51 +0800 Subject: [PATCH 09/11] address comment --- lightning/config/config.go | 8 +- lightning/mydump/parser_generated.go | 113 +++++---- lightning/restore/checkpoints.go | 55 ++--- lightning/restore/file_checkpoints.pb.go | 301 +++++++++++++++-------- lightning/restore/file_checkpoints.proto | 2 +- lightning/restore/restore.go | 106 ++++---- tests/checkpoint/run.sh | 25 +- 7 files changed, 375 insertions(+), 235 deletions(-) diff --git a/lightning/config/config.go b/lightning/config/config.go index 4ffc3b2c9..ea7067db5 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -145,6 +145,7 @@ func NewConfig() *Config { App: Lightning{ RegionConcurrency: runtime.NumCPU(), TableConcurrency: 8, + IndexConcurrency: 2, IOConcurrency: 5, CheckRequirements: true, }, @@ -235,12 +236,5 @@ func (cfg *Config) Load() error { cfg.PostRestore.Level1Compact = new(bool) *cfg.PostRestore.Level1Compact = false } - - if cfg.App.TableConcurrency < 2 { - cfg.App.TableConcurrency = 2 - } - if cfg.App.IndexConcurrency < 2 { - cfg.App.IndexConcurrency = 2 - } return nil } diff --git a/lightning/mydump/parser_generated.go b/lightning/mydump/parser_generated.go index 0e0ee4d42..f3578797c 100644 --- a/lightning/mydump/parser_generated.go +++ b/lightning/mydump/parser_generated.go @@ -1,6 +1,19 @@ // Code generated by ragel DO NOT EDIT. //.... lightning/mydump/parser.rl:1 +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + // Please edit `parser.rl` if you want to modify this file. To generate // `parser_generated.go`, please execute // @@ -17,21 +30,21 @@ import ( "github.com/pingcap/tidb-lightning/lightning/common" ) -//.... lightning/mydump/parser.rl:79 +//.... lightning/mydump/parser.rl:92 -//.... tmp_parser.go:25 +//.... tmp_parser.go:38 const chunk_parser_start int = 27 const chunk_parser_first_final int = 27 const chunk_parser_error int = 0 const chunk_parser_en_main int = 27 -//.... lightning/mydump/parser.rl:82 +//.... lightning/mydump/parser.rl:95 func (parser *ChunkParser) lex() (token, []byte, error) { var cs, ts, te, act, p int - //.... tmp_parser.go:38 + //.... tmp_parser.go:51 { cs = chunk_parser_start ts = 0 @@ -39,7 +52,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { act = 0 } - //.... lightning/mydump/parser.rl:86 + //.... lightning/mydump/parser.rl:99 for { data := parser.buf @@ -50,7 +63,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { eof = pe } - //.... tmp_parser.go:58 + //.... tmp_parser.go:71 { if p == pe { goto _test_eof @@ -191,7 +204,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto st27 tr9: - //.... lightning/mydump/parser.rl:67 + //.... lightning/mydump/parser.rl:80 te = p + 1 { consumedToken = tokRow @@ -203,7 +216,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { } goto st27 tr19: - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 p = (te) - 1 { consumedToken = tokName @@ -215,12 +228,12 @@ func (parser *ChunkParser) lex() (token, []byte, error) { } goto st27 tr21: - //.... lightning/mydump/parser.rl:60 + //.... lightning/mydump/parser.rl:73 te = p + 1 goto st27 tr40: - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 te = p p-- { @@ -233,7 +246,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { } goto st27 tr41: - //.... lightning/mydump/parser.rl:60 + //.... lightning/mydump/parser.rl:73 te = p p-- @@ -252,7 +265,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 ts = p - //.... tmp_parser.go:233 + //.... tmp_parser.go:246 switch data[p] { case 32: goto tr21 @@ -292,21 +305,21 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st28 tr43: //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:60 + //.... lightning/mydump/parser.rl:73 act = 1 goto st28 tr53: //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:62 + //.... lightning/mydump/parser.rl:75 act = 2 goto st28 st28: @@ -314,7 +327,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof28 } st_case_28: - //.... tmp_parser.go:295 + //.... tmp_parser.go:308 switch data[p] { case 32: goto tr0 @@ -497,7 +510,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st29 st29: @@ -505,7 +518,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof29 } st_case_29: - //.... tmp_parser.go:486 + //.... tmp_parser.go:499 switch data[p] { case 32: goto tr40 @@ -539,7 +552,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof30 } st_case_30: - //.... tmp_parser.go:520 + //.... tmp_parser.go:533 switch data[p] { case 10: goto tr21 @@ -590,7 +603,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:60 + //.... lightning/mydump/parser.rl:73 act = 1 goto st31 st31: @@ -598,7 +611,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof31 } st_case_31: - //.... tmp_parser.go:579 + //.... tmp_parser.go:592 switch data[p] { case 34: goto tr2 @@ -631,7 +644,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:60 + //.... lightning/mydump/parser.rl:73 act = 1 goto st32 st32: @@ -639,7 +652,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof32 } st_case_32: - //.... tmp_parser.go:620 + //.... tmp_parser.go:633 if data[p] == 96 { goto tr2 } @@ -648,7 +661,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st33 st33: @@ -656,7 +669,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof33 } st_case_33: - //.... tmp_parser.go:637 + //.... tmp_parser.go:650 switch data[p] { case 32: goto tr40 @@ -690,7 +703,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof34 } st_case_34: - //.... tmp_parser.go:671 + //.... tmp_parser.go:684 switch data[p] { case 32: goto st20 @@ -784,7 +797,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof35 } st_case_35: - //.... tmp_parser.go:765 + //.... tmp_parser.go:778 switch data[p] { case 32: goto st20 @@ -840,7 +853,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st36 st36: @@ -848,7 +861,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof36 } st_case_36: - //.... tmp_parser.go:829 + //.... tmp_parser.go:842 switch data[p] { case 32: goto tr40 @@ -878,7 +891,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st37 st37: @@ -886,7 +899,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof37 } st_case_37: - //.... tmp_parser.go:867 + //.... tmp_parser.go:880 switch data[p] { case 32: goto tr40 @@ -920,7 +933,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st38 st38: @@ -928,7 +941,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof38 } st_case_38: - //.... tmp_parser.go:909 + //.... tmp_parser.go:922 switch data[p] { case 32: goto tr40 @@ -958,7 +971,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st39 st39: @@ -966,7 +979,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof39 } st_case_39: - //.... tmp_parser.go:947 + //.... tmp_parser.go:960 switch data[p] { case 32: goto tr40 @@ -996,7 +1009,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st40 st40: @@ -1004,7 +1017,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof40 } st_case_40: - //.... tmp_parser.go:985 + //.... tmp_parser.go:998 switch data[p] { case 32: goto tr40 @@ -1034,7 +1047,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st41 st41: @@ -1042,7 +1055,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof41 } st_case_41: - //.... tmp_parser.go:1023 + //.... tmp_parser.go:1036 switch data[p] { case 32: goto tr40 @@ -1072,7 +1085,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st42 st42: @@ -1080,7 +1093,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof42 } st_case_42: - //.... tmp_parser.go:1061 + //.... tmp_parser.go:1074 switch data[p] { case 32: goto tr40 @@ -1110,7 +1123,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st43 st43: @@ -1118,7 +1131,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof43 } st_case_43: - //.... tmp_parser.go:1099 + //.... tmp_parser.go:1112 switch data[p] { case 32: goto tr40 @@ -1148,7 +1161,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st44 st44: @@ -1156,7 +1169,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof44 } st_case_44: - //.... tmp_parser.go:1137 + //.... tmp_parser.go:1150 switch data[p] { case 32: goto tr40 @@ -1186,7 +1199,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st45 st45: @@ -1194,7 +1207,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof45 } st_case_45: - //.... tmp_parser.go:1175 + //.... tmp_parser.go:1188 switch data[p] { case 32: goto tr40 @@ -1224,7 +1237,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { //.... NONE:1 te = p + 1 - //.... lightning/mydump/parser.rl:72 + //.... lightning/mydump/parser.rl:85 act = 4 goto st46 st46: @@ -1232,7 +1245,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { goto _test_eof46 } st_case_46: - //.... tmp_parser.go:1213 + //.... tmp_parser.go:1226 switch data[p] { case 32: goto tr40 @@ -1477,7 +1490,7 @@ func (parser *ChunkParser) lex() (token, []byte, error) { } } - //.... lightning/mydump/parser.rl:97 + //.... lightning/mydump/parser.rl:110 if cs == 0 { common.AppLogger.Errorf("Syntax error near byte %d, content is «%s»", parser.pos, string(data)) diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index a29f98941..c8088970e 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -117,7 +117,7 @@ type EngineCheckpoint struct { type TableCheckpoint struct { Status CheckpointStatus AllocBase int64 - Engines []*EngineCheckpoint + Engines map[int]*EngineCheckpoint } func (cp *TableCheckpoint) CountChunks() int { @@ -185,7 +185,7 @@ type TableCheckpointMerger interface { } type StatusCheckpointMerger struct { - EngineID int // -1 == apply to whole table. + EngineID int // wholeTableEngineID == apply to whole table. Status CheckpointStatus } @@ -194,7 +194,7 @@ func (merger *StatusCheckpointMerger) SetInvalid() { } func (merger *StatusCheckpointMerger) MergeInto(cpd *TableCheckpointDiff) { - if merger.EngineID == invalidEngineID || merger.Status <= CheckpointStatusMaxInvalid { + if merger.EngineID == wholeTableEngineID || merger.Status <= CheckpointStatusMaxInvalid { cpd.status = merger.Status cpd.hasStatus = true } @@ -245,7 +245,7 @@ type CheckpointsDB interface { Initialize(ctx context.Context, dbInfo map[string]*TidbDBInfo) error Get(ctx context.Context, tableName string) (*TableCheckpoint, error) Close() error - InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints []*EngineCheckpoint) error + InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error Update(checkpointDiffs map[string]*TableCheckpointDiff) RemoveCheckpoint(ctx context.Context, tableName string) error @@ -272,11 +272,12 @@ func (*NullCheckpointsDB) Close() error { func (*NullCheckpointsDB) Get(_ context.Context, _ string) (*TableCheckpoint, error) { return &TableCheckpoint{ - Status: CheckpointStatusLoaded, + Status: CheckpointStatusLoaded, + Engines: map[int]*EngineCheckpoint{}, }, nil } -func (*NullCheckpointsDB) InsertEngineCheckpoints(_ context.Context, _ string, _ []*EngineCheckpoint) error { +func (*NullCheckpointsDB) InsertEngineCheckpoints(_ context.Context, _ string, _ map[int]*EngineCheckpoint) error { return nil } @@ -412,7 +413,9 @@ func (cpdb *MySQLCheckpointsDB) Close() error { } func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*TableCheckpoint, error) { - cp := new(TableCheckpoint) + cp := &TableCheckpoint{ + Engines: map[int]*EngineCheckpoint{}, + } purpose := "(read checkpoint " + tableName + ")" err := common.TransactWithRetry(ctx, cpdb.db, purpose, func(c context.Context, tx *sql.Tx) error { @@ -434,17 +437,9 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab if err := engineRows.Scan(&engineID, &status); err != nil { return errors.Trace(err) } - var found bool - for _, engine := range cp.Engines { - if engineID == engine.EngineID { - engine.Status = CheckpointStatus(status) - found = true - break - } - } - if !found { - checkpoint := &EngineCheckpoint{EngineID: engineID, Status: CheckpointStatus(status)} - cp.Engines = append(cp.Engines, checkpoint) + cp.Engines[engineID] = &EngineCheckpoint{ + EngineID: engineID, + Status: CheckpointStatus(status), } } if err := engineRows.Err(); err != nil { @@ -482,8 +477,6 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab return errors.Trace(err) } value.Checksum = verify.MakeKVChecksum(kvcBytes, kvcKVs, kvcChecksum) - // It is ok to use engineID here, because index engine file will not - // contains any chunks cp.Engines[engineID].Chunks = append(cp.Engines[engineID].Chunks, value) } if err := chunkRows.Err(); err != nil { @@ -511,7 +504,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab return cp, nil } -func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints []*EngineCheckpoint) error { +func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error { err := common.TransactWithRetry(ctx, cpdb.db, "(update engine checkpoints for "+tableName+")", func(c context.Context, tx *sql.Tx) error { engineStmt, err := tx.PrepareContext(c, fmt.Sprintf(` REPLACE INTO %s.%s (table_name, engine_id, status) VALUES (?, ?, ?); @@ -646,7 +639,12 @@ type FileCheckpointsDB struct { } func NewFileCheckpointsDB(path string) *FileCheckpointsDB { - cpdb := &FileCheckpointsDB{path: path} + cpdb := &FileCheckpointsDB{ + path: path, + checkpoints: CheckpointsModel{ + Checkpoints: map[string]*TableCheckpointModel{}, + }, + } // ignore all errors -- file maybe not created yet (and it is fine). content, err := ioutil.ReadFile(path) if err == nil { @@ -681,7 +679,8 @@ func (cpdb *FileCheckpointsDB) Initialize(ctx context.Context, dbInfo map[string tableName := common.UniqueTable(db.Name, table.Name) if _, ok := cpdb.checkpoints.Checkpoints[tableName]; !ok { cpdb.checkpoints.Checkpoints[tableName] = &TableCheckpointModel{ - Status: uint32(CheckpointStatusLoaded), + Status: uint32(CheckpointStatusLoaded), + Engines: map[int32]*EngineCheckpointModel{}, } } // TODO check if hash matches @@ -707,7 +706,7 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC cp := &TableCheckpoint{ Status: CheckpointStatus(tableModel.Status), AllocBase: tableModel.AllocBase, - Engines: make([]*EngineCheckpoint, 0, len(tableModel.Engines)), + Engines: make(map[int]*EngineCheckpoint, len(tableModel.Engines)), } for _, engineModel := range tableModel.Engines { @@ -739,13 +738,13 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC return engine.Chunks[i].Key.less(&engine.Chunks[j].Key) }) - cp.Engines = append(cp.Engines, engine) + cp.Engines[engine.EngineID] = engine } return cp, nil } -func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableName string, checkpoints []*EngineCheckpoint) error { +func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error { cpdb.lock.Lock() defer cpdb.lock.Unlock() @@ -776,7 +775,7 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk.KvcKvs = value.Checksum.SumKVS() chunk.KvcChecksum = value.Checksum.Sum() } - tableModel.Engines = append(tableModel.Engines, engineModel) + tableModel.Engines[int32(engine.EngineID)] = engineModel } return errors.Trace(cpdb.save()) @@ -795,7 +794,7 @@ func (cpdb *FileCheckpointsDB) Update(checkpointDiffs map[string]*TableCheckpoin tableModel.AllocBase = cpd.allocBase } for engineID, engineDiff := range cpd.engines { - engineModel := tableModel.Engines[engineID] + engineModel := tableModel.Engines[int32(engineID)] if engineDiff.hasStatus { engineModel.Status = uint32(engineDiff.status) } diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index 82659a77a..8174f4423 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -62,11 +62,11 @@ func (m *CheckpointsModel) XXX_DiscardUnknown() { var xxx_messageInfo_CheckpointsModel proto.InternalMessageInfo type TableCheckpointModel struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` - Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` - AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` - Engines []*EngineCheckpointModel `protobuf:"bytes,6,rep,name=engines,proto3" json:"engines,omitempty"` - IndexEngineStatuses []uint32 `protobuf:"varint,7,rep,packed,name=index_engine_statuses,json=indexEngineStatuses,proto3" json:"index_engine_statuses,omitempty"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` + AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` + Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + IndexEngineStatuses []uint32 `protobuf:"varint,7,rep,packed,name=index_engine_statuses,json=indexEngineStatuses,proto3" json:"index_engine_statuses,omitempty"` } func (m *TableCheckpointModel) Reset() { *m = TableCheckpointModel{} } @@ -193,6 +193,7 @@ func init() { proto.RegisterType((*CheckpointsModel)(nil), "CheckpointsModel") proto.RegisterMapType((map[string]*TableCheckpointModel)(nil), "CheckpointsModel.CheckpointsEntry") proto.RegisterType((*TableCheckpointModel)(nil), "TableCheckpointModel") + proto.RegisterMapType((map[int32]*EngineCheckpointModel)(nil), "TableCheckpointModel.EnginesEntry") proto.RegisterType((*EngineCheckpointModel)(nil), "EngineCheckpointModel") proto.RegisterMapType((map[string]*ChunkCheckpointModel)(nil), "EngineCheckpointModel.ChunksEntry") proto.RegisterType((*ChunkCheckpointModel)(nil), "ChunkCheckpointModel") @@ -203,45 +204,46 @@ func init() { } var fileDescriptor_c47ec4f2f281cd62 = []byte{ - // 597 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xcd, 0xc4, 0x6d, 0x7e, 0x6e, 0xd2, 0x4f, 0xd5, 0x7c, 0x4d, 0x19, 0x15, 0x61, 0x99, 0x08, - 0x24, 0x4b, 0x88, 0x04, 0xca, 0x06, 0x75, 0xd9, 0xd2, 0x45, 0x85, 0x2a, 0xd0, 0x00, 0x1b, 0x36, - 0x96, 0x63, 0x4f, 0x6c, 0xcb, 0xce, 0x4c, 0xe4, 0xb1, 0xdd, 0xf6, 0x2d, 0x78, 0x13, 0x1e, 0x83, - 0x2e, 0xbb, 0x44, 0x88, 0x05, 0xb4, 0x2f, 0x82, 0x66, 0xc6, 0x55, 0xdc, 0x2a, 0x42, 0xec, 0xee, - 0x3d, 0xe7, 0xdc, 0x33, 0x77, 0x7c, 0x34, 0x06, 0x37, 0x4b, 0xa2, 0xb8, 0xe0, 0x09, 0x8f, 0xa6, - 0x39, 0x93, 0x85, 0xc8, 0xd9, 0x74, 0x9e, 0x64, 0xcc, 0x0b, 0x62, 0x16, 0xa4, 0x4b, 0x91, 0xf0, - 0x42, 0x4e, 0x96, 0xb9, 0x28, 0xc4, 0xde, 0xf3, 0x28, 0x29, 0xe2, 0x72, 0x36, 0x09, 0xc4, 0x62, - 0x1a, 0x89, 0x48, 0x4c, 0x35, 0x3c, 0x2b, 0xe7, 0xba, 0xd3, 0x8d, 0xae, 0x8c, 0x7c, 0xfc, 0x15, - 0xc1, 0xf6, 0xd1, 0xca, 0xe4, 0x54, 0x84, 0x2c, 0xc3, 0x6f, 0x60, 0xd0, 0x30, 0x26, 0xc8, 0xb1, - 0xdc, 0xc1, 0xfe, 0x78, 0x72, 0x5f, 0xd7, 0x04, 0x8e, 0x79, 0x91, 0x5f, 0xd0, 0xe6, 0xd8, 0xde, - 0xa7, 0x3b, 0xce, 0x5a, 0x80, 0xb7, 0xc1, 0x4a, 0xd9, 0x05, 0x41, 0x0e, 0x72, 0xfb, 0x54, 0x95, - 0xf8, 0x19, 0x6c, 0x56, 0x7e, 0x56, 0x32, 0xd2, 0x76, 0x90, 0x3b, 0xd8, 0x1f, 0x4d, 0x3e, 0xfa, - 0xb3, 0x8c, 0xad, 0x06, 0xf5, 0x49, 0xd4, 0x68, 0x0e, 0xda, 0xaf, 0xd1, 0xf8, 0x1b, 0x82, 0x9d, - 0x75, 0x1a, 0x8c, 0x61, 0x23, 0xf6, 0x65, 0xac, 0xcd, 0x87, 0x54, 0xd7, 0x78, 0x17, 0x3a, 0xb2, - 0xf0, 0x8b, 0x52, 0x12, 0xcb, 0x41, 0xee, 0x16, 0xad, 0x3b, 0xfc, 0x08, 0xc0, 0xcf, 0x32, 0x11, - 0x78, 0x33, 0x5f, 0x32, 0xb2, 0xe1, 0x20, 0xd7, 0xa2, 0x7d, 0x8d, 0x1c, 0xfa, 0x92, 0xe1, 0x17, - 0xd0, 0x65, 0x3c, 0x4a, 0x38, 0x93, 0xa4, 0xa3, 0x2f, 0xbf, 0x3b, 0x39, 0xd6, 0xfd, 0xfd, 0xbd, - 0x6e, 0x65, 0x78, 0x1f, 0x46, 0x09, 0x0f, 0xd9, 0xb9, 0x67, 0x00, 0xcf, 0x9c, 0xc3, 0x24, 0xe9, - 0x3a, 0x96, 0xbb, 0x45, 0xff, 0xd7, 0xa4, 0xf1, 0xf8, 0x50, 0x53, 0xe3, 0x1f, 0x08, 0x46, 0x6b, - 0x6d, 0x1b, 0x6b, 0xa3, 0x3b, 0x6b, 0x1f, 0x40, 0x27, 0x88, 0x4b, 0x9e, 0x4a, 0xd2, 0xae, 0x33, - 0x59, 0x3b, 0x3f, 0x39, 0xd2, 0x22, 0x93, 0x49, 0x3d, 0x81, 0x1f, 0x42, 0xbf, 0xde, 0x2d, 0x09, - 0xf5, 0xd7, 0xd8, 0xa4, 0x3d, 0x03, 0x9c, 0x84, 0x7b, 0xef, 0x61, 0xd0, 0x98, 0xf9, 0x97, 0x98, - 0xb4, 0xfc, 0x2f, 0x31, 0xfd, 0x6c, 0xc3, 0xce, 0x3a, 0x8d, 0x8a, 0x69, 0xe9, 0x17, 0x71, 0x6d, - 0xae, 0x6b, 0x75, 0x5f, 0x31, 0x9f, 0x4b, 0x56, 0x68, 0x7b, 0x8b, 0xd6, 0x1d, 0x26, 0xd0, 0x0d, - 0x44, 0x56, 0x2e, 0xb8, 0xc9, 0x6f, 0x48, 0x6f, 0x5b, 0xfc, 0x12, 0x46, 0x32, 0x16, 0x65, 0x16, - 0x7a, 0x09, 0x0f, 0xb2, 0x32, 0x64, 0x5e, 0x2e, 0xce, 0xd4, 0xcd, 0x54, 0x96, 0x3d, 0x8a, 0x0d, - 0x79, 0x62, 0x38, 0x2a, 0xce, 0x4e, 0x42, 0x95, 0x39, 0xe3, 0xa1, 0x57, 0x1f, 0xb4, 0x69, 0x32, - 0x67, 0x3c, 0x7c, 0x67, 0xce, 0xda, 0x06, 0x6b, 0x29, 0x54, 0xde, 0x0a, 0x57, 0x25, 0x7e, 0x02, - 0xff, 0x2d, 0x73, 0x56, 0x29, 0xe7, 0x24, 0xf4, 0x16, 0xfe, 0x39, 0xe9, 0x6a, 0x72, 0xa8, 0x50, - 0xaa, 0xc0, 0x53, 0xff, 0x5c, 0x7d, 0xd7, 0x95, 0xa0, 0xa7, 0x05, 0xbd, 0xbc, 0x41, 0xa6, 0x55, - 0xe0, 0xcd, 0x2e, 0x0a, 0x26, 0x49, 0xdf, 0x41, 0xee, 0x06, 0xed, 0xa5, 0x55, 0x70, 0xa8, 0x7a, - 0xfc, 0x00, 0xba, 0x8a, 0x4c, 0x2b, 0x49, 0x40, 0x53, 0x9d, 0xb4, 0x0a, 0xde, 0x56, 0x12, 0x3f, - 0x86, 0xa1, 0x22, 0xf4, 0x63, 0x92, 0xe5, 0x82, 0x0c, 0x1c, 0xe4, 0x76, 0xe8, 0x20, 0xad, 0x82, - 0xa3, 0x1a, 0x3a, 0x7c, 0x7a, 0xf9, 0xdb, 0x6e, 0x5d, 0x5e, 0xdb, 0xe8, 0xea, 0xda, 0x46, 0xbf, - 0xae, 0x6d, 0xf4, 0xe5, 0xc6, 0x6e, 0x5d, 0xdd, 0xd8, 0xad, 0xef, 0x37, 0x76, 0xeb, 0x73, 0xb7, - 0xfe, 0x49, 0xcc, 0x3a, 0xfa, 0x95, 0xbf, 0xfa, 0x13, 0x00, 0x00, 0xff, 0xff, 0x74, 0x46, 0x29, - 0xb6, 0x40, 0x04, 0x00, 0x00, + // 617 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xcd, 0xc6, 0xcd, 0xd7, 0x24, 0x45, 0xd5, 0xd2, 0x94, 0x55, 0x10, 0x96, 0xa9, 0x40, 0xb2, + 0x04, 0x24, 0xa2, 0x5c, 0x50, 0xc5, 0xa9, 0xa5, 0x87, 0x0a, 0x55, 0xa0, 0x05, 0x2e, 0x5c, 0x2c, + 0xc7, 0xde, 0xc4, 0x96, 0x1d, 0x6f, 0xe4, 0xb5, 0xdd, 0xf6, 0x5f, 0xf0, 0x2f, 0x38, 0xf2, 0x37, + 0x7a, 0xec, 0x11, 0x21, 0x0e, 0xd0, 0xfe, 0x11, 0xb4, 0x1f, 0x55, 0x1d, 0x14, 0x55, 0xdc, 0x76, + 0xde, 0x7b, 0x33, 0xa3, 0xf7, 0x34, 0x36, 0xb8, 0x69, 0x3c, 0x8f, 0x8a, 0x2c, 0xce, 0xe6, 0x93, + 0x9c, 0x89, 0x82, 0xe7, 0x6c, 0x32, 0x8b, 0x53, 0xe6, 0x05, 0x11, 0x0b, 0x92, 0x25, 0x8f, 0xb3, + 0x42, 0x8c, 0x97, 0x39, 0x2f, 0xf8, 0xe8, 0xc5, 0x3c, 0x2e, 0xa2, 0x72, 0x3a, 0x0e, 0xf8, 0x62, + 0x32, 0xe7, 0x73, 0x3e, 0x51, 0xf0, 0xb4, 0x9c, 0xa9, 0x4a, 0x15, 0xea, 0xa5, 0xe5, 0xbb, 0xdf, + 0x11, 0x6c, 0x1d, 0xde, 0x0e, 0x39, 0xe1, 0x21, 0x4b, 0xf1, 0x5b, 0xe8, 0xd7, 0x06, 0x13, 0xe4, + 0x58, 0x6e, 0x7f, 0x6f, 0x77, 0xfc, 0xaf, 0xae, 0x0e, 0x1c, 0x65, 0x45, 0x7e, 0x4e, 0xeb, 0x6d, + 0xa3, 0xcf, 0x2b, 0x93, 0x95, 0x00, 0x6f, 0x81, 0x95, 0xb0, 0x73, 0x82, 0x1c, 0xe4, 0xf6, 0xa8, + 0x7c, 0xe2, 0x67, 0xd0, 0xaa, 0xfc, 0xb4, 0x64, 0xa4, 0xe9, 0x20, 0xb7, 0xbf, 0x37, 0x1c, 0x7f, + 0xf2, 0xa7, 0x29, 0xbb, 0x6d, 0x54, 0x9b, 0xa8, 0xd6, 0xec, 0x37, 0x5f, 0xa3, 0xdd, 0x6f, 0x4d, + 0xd8, 0x5e, 0xa7, 0xc1, 0x18, 0x36, 0x22, 0x5f, 0x44, 0x6a, 0xf8, 0x80, 0xaa, 0x37, 0xde, 0x81, + 0xb6, 0x28, 0xfc, 0xa2, 0x14, 0xc4, 0x72, 0x90, 0xbb, 0x49, 0x4d, 0x85, 0x1f, 0x01, 0xf8, 0x69, + 0xca, 0x03, 0x6f, 0xea, 0x0b, 0x46, 0x36, 0x1c, 0xe4, 0x5a, 0xb4, 0xa7, 0x90, 0x03, 0x5f, 0x30, + 0xfc, 0x06, 0x3a, 0x2c, 0x9b, 0xc7, 0x19, 0x13, 0xa4, 0x6b, 0xcc, 0xaf, 0x5b, 0x39, 0x3e, 0xd2, + 0x22, 0x6d, 0xfe, 0xa6, 0x05, 0xef, 0xc1, 0x30, 0xce, 0x42, 0x76, 0xe6, 0x69, 0xc0, 0xd3, 0x3b, + 0x99, 0x20, 0x1d, 0xc7, 0x72, 0x37, 0xe9, 0x7d, 0x45, 0xea, 0xd6, 0x8f, 0x86, 0x1a, 0x51, 0x18, + 0xd4, 0x87, 0xd5, 0x83, 0x6a, 0xe9, 0xa0, 0x9e, 0xaf, 0x06, 0xb5, 0x63, 0x96, 0xdf, 0x91, 0xd4, + 0x4f, 0x04, 0xc3, 0xb5, 0xa2, 0x5a, 0x2c, 0x68, 0x25, 0x96, 0x7d, 0x68, 0x07, 0x51, 0x99, 0x25, + 0x82, 0x34, 0x8d, 0xed, 0xb5, 0xfd, 0xe3, 0x43, 0x25, 0xd2, 0xb6, 0x4d, 0x07, 0x7e, 0x08, 0x3d, + 0xe3, 0x37, 0x0e, 0x55, 0xda, 0x2d, 0xda, 0xd5, 0xc0, 0x71, 0x38, 0xfa, 0x00, 0xfd, 0x5a, 0xcf, + 0xff, 0x9c, 0x81, 0x92, 0xdf, 0x61, 0xee, 0x57, 0x13, 0xb6, 0xd7, 0x69, 0xe4, 0x19, 0x2c, 0xfd, + 0x22, 0x32, 0xc3, 0xd5, 0x5b, 0xfa, 0xe5, 0xb3, 0x99, 0x60, 0x85, 0x1a, 0x6f, 0x51, 0x53, 0x61, + 0x02, 0x9d, 0x80, 0xa7, 0xe5, 0x22, 0xd3, 0xf7, 0x31, 0xa0, 0x37, 0x25, 0x7e, 0x09, 0x43, 0x11, + 0xf1, 0x32, 0x0d, 0xbd, 0x38, 0x0b, 0xd2, 0x32, 0x64, 0x5e, 0xce, 0x4f, 0xa5, 0x33, 0x79, 0x2b, + 0x5d, 0x8a, 0x35, 0x79, 0xac, 0x39, 0xca, 0x4f, 0x8f, 0x43, 0x79, 0x53, 0x2c, 0x0b, 0x3d, 0xb3, + 0xa8, 0xa5, 0x6f, 0x8a, 0x65, 0xe1, 0x7b, 0xbd, 0x6b, 0x0b, 0xac, 0x25, 0x17, 0xa4, 0xad, 0x70, + 0xf9, 0xc4, 0x4f, 0xe0, 0xde, 0x32, 0x67, 0x95, 0x9c, 0x1c, 0x87, 0xde, 0xc2, 0x3f, 0x23, 0x1d, + 0x45, 0x0e, 0x24, 0x4a, 0x25, 0x78, 0xe2, 0x9f, 0xc9, 0x5c, 0x6f, 0x05, 0x5d, 0x25, 0xe8, 0xe6, + 0x35, 0x32, 0xa9, 0x02, 0x6f, 0x7a, 0x5e, 0x30, 0x41, 0x7a, 0x0e, 0x72, 0x37, 0x68, 0x37, 0xa9, + 0x82, 0x03, 0x59, 0xe3, 0x07, 0xd0, 0x91, 0x64, 0x52, 0x09, 0x02, 0x8a, 0x6a, 0x27, 0x55, 0xf0, + 0xae, 0x12, 0xf8, 0x31, 0x0c, 0x24, 0xa1, 0x3e, 0x56, 0x51, 0x2e, 0x48, 0xdf, 0x41, 0x6e, 0x9b, + 0xf6, 0x93, 0x2a, 0x38, 0x34, 0xd0, 0xc1, 0xd3, 0x8b, 0x3f, 0x76, 0xe3, 0xe2, 0xca, 0x46, 0x97, + 0x57, 0x36, 0xfa, 0x7d, 0x65, 0xa3, 0xaf, 0xd7, 0x76, 0xe3, 0xf2, 0xda, 0x6e, 0xfc, 0xb8, 0xb6, + 0x1b, 0x5f, 0x3a, 0xe6, 0x27, 0x34, 0x6d, 0xab, 0xbf, 0xc8, 0xab, 0xbf, 0x01, 0x00, 0x00, 0xff, + 0xff, 0x6e, 0x1a, 0x8d, 0x1b, 0xa0, 0x04, 0x00, 0x00, } func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { @@ -321,18 +323,6 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(m.AllocBase)) } - if len(m.Engines) > 0 { - for _, msg := range m.Engines { - dAtA[i] = 0x32 - i++ - i = encodeVarintFileCheckpoints(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } - } if len(m.IndexEngineStatuses) > 0 { dAtA3 := make([]byte, len(m.IndexEngineStatuses)*10) var j2 int @@ -350,6 +340,33 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintFileCheckpoints(dAtA, i, uint64(j2)) i += copy(dAtA[i:], dAtA3[:j2]) } + if len(m.Engines) > 0 { + for k, _ := range m.Engines { + dAtA[i] = 0x42 + i++ + v := m.Engines[k] + msgSize := 0 + if v != nil { + msgSize = v.Size() + msgSize += 1 + sovFileCheckpoints(uint64(msgSize)) + } + mapSize := 1 + sovFileCheckpoints(uint64(k)) + msgSize + i = encodeVarintFileCheckpoints(dAtA, i, uint64(mapSize)) + dAtA[i] = 0x8 + i++ + i = encodeVarintFileCheckpoints(dAtA, i, uint64(k)) + if v != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintFileCheckpoints(dAtA, i, uint64(v.Size())) + n4, err := v.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + } + } return i, nil } @@ -393,11 +410,11 @@ func (m *EngineCheckpointModel) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(v.Size())) - n4, err := v.MarshalTo(dAtA[i:]) + n5, err := v.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n4 + i += n5 } } } @@ -537,12 +554,6 @@ func (m *TableCheckpointModel) Size() (n int) { if m.AllocBase != 0 { n += 1 + sovFileCheckpoints(uint64(m.AllocBase)) } - if len(m.Engines) > 0 { - for _, e := range m.Engines { - l = e.Size() - n += 1 + l + sovFileCheckpoints(uint64(l)) - } - } if len(m.IndexEngineStatuses) > 0 { l = 0 for _, e := range m.IndexEngineStatuses { @@ -550,6 +561,19 @@ func (m *TableCheckpointModel) Size() (n int) { } n += 1 + sovFileCheckpoints(uint64(l)) + l } + if len(m.Engines) > 0 { + for k, v := range m.Engines { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovFileCheckpoints(uint64(l)) + } + mapEntrySize := 1 + sovFileCheckpoints(uint64(k)) + l + n += mapEntrySize + 1 + sovFileCheckpoints(uint64(mapEntrySize)) + } + } return n } @@ -921,40 +945,6 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { break } } - case 6: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Engines", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthFileCheckpoints - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Engines = append(m.Engines, &EngineCheckpointModel{}) - if err := m.Engines[len(m.Engines)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 7: if wireType == 0 { var v uint32 @@ -1031,6 +1021,121 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field IndexEngineStatuses", wireType) } + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Engines", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthFileCheckpoints + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Engines == nil { + m.Engines = make(map[int32]*EngineCheckpointModel) + } + var mapkey int32 + var mapvalue *EngineCheckpointModel + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapkey |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileCheckpoints + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthFileCheckpoints + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthFileCheckpoints + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &EngineCheckpointModel{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthFileCheckpoints + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Engines[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index 6ae031312..183bf475f 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -27,7 +27,7 @@ message TableCheckpointModel { bytes hash = 1; uint32 status = 3; int64 alloc_base = 4; - repeated EngineCheckpointModel engines = 6; + map engines = 8; repeated uint32 index_engine_statuses = 7; } diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index e3107963b..640095c0c 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -57,8 +57,8 @@ const ( ) const ( - indexEngineID = -1 - invalidEngineID = math.MinInt64 + indexEngineID = -1 + wholeTableEngineID = math.MinInt64 ) const ( @@ -348,42 +348,32 @@ func (rc *RestoreController) listenCheckpointUpdates(wg *sync.WaitGroup) { lock.Unlock() - // Note: about gofailLabel1, gofailLabel2, etc.., just a hack way - // for avoiding generated error message e.g: - // `failpoint: "github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported" got value 140 of type "int" but expected type "int"` - // gofail: var FailIfImportedChunk struct{} // if _, ok := scp.merger.(*ChunkCheckpointMerger); ok { // wg.Wait() // panic("forcing failure due to FailIfImportedChunk") // } - // goto gofailLabel1 - gofailLabel1: - if false { - goto gofailLabel1 - } + // goto RETURN1 + + // gofail: RETURN1: // gofail: var FailIfStatusBecomes int // if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == FailIfStatusBecomes { // wg.Wait() // panic("forcing failure due to FailIfStatusBecomes") // } - // goto gofailLabel2 - gofailLabel2: - if false { - goto gofailLabel2 - } + // goto RETURN2 + + // gofail: RETURN2: // gofail: var FailIfIndexEngineImported int - // if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID == invalidEngineID && merger.Status == CheckpointStatusIndexImported && FailIfIndexEngineImported > 0 { + // if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID == wholeTableEngineID && merger.Status == CheckpointStatusIndexImported && FailIfIndexEngineImported > 0 { // wg.Wait() // panic("forcing failure due to FailIfIndexEngineImported") // } - // goto gofailLabel3 - gofailLabel3: - if false { - goto gofailLabel3 - } + // goto RETURN3 + + // gofail: RETURN3: } } @@ -546,17 +536,20 @@ func (t *TableRestore) restoreTable( // The table checkpoint status set to `CheckpointStatusIndexImported` only if // both all data engines and the index engine had been imported to TiKV. + // But persist index engine checkpoint status and table checkpoint status are + // not an atomic operation, so `cp.Status < CheckpointStatusIndexImported` + // but `indexEngineCp.Status == CheckpointStatusImported` could happen + // when kill lightning after saving index engine checkpoint status before saving + // table checkpoint status. var indexEngine *kv.OpenedEngine var closedIndexEngine *kv.ClosedEngine - if cp.Status < CheckpointStatusIndexImported { - if indexEngineCp.Status < CheckpointStatusImported { - indexWorker := rc.indexWorkers.Apply() - defer rc.indexWorkers.Recycle(indexWorker) - var err error - indexEngine, err = rc.importer.OpenEngine(ctx, t.tableName, indexEngineID) - if err != nil { - return errors.Trace(err) - } + if indexEngineCp.Status < CheckpointStatusImported && cp.Status < CheckpointStatusIndexImported { + indexWorker := rc.indexWorkers.Apply() + defer rc.indexWorkers.Recycle(indexWorker) + var err error + indexEngine, err = rc.importer.OpenEngine(ctx, t.tableName, indexEngineID) + if err != nil { + return errors.Trace(err) } // The table checkpoint status less than `CheckpointStatusIndexImported` implies @@ -614,7 +607,7 @@ func (t *TableRestore) restoreTable( wg.Wait() common.AppLogger.Infof("[%s] import whole table takes %v", t.tableName, time.Since(timer)) - err := engineErr.Get() + err = engineErr.Get() if err != nil { return errors.Trace(err) } @@ -634,7 +627,7 @@ func (t *TableRestore) restoreTable( } // 3. Post-process - return errors.Trace(t.postProcess(ctx, rc, cp, closedIndexEngine)) + return errors.Trace(t.postProcess(ctx, rc, cp, indexEngineCp, closedIndexEngine)) } func (t *TableRestore) restoreEngine( @@ -784,14 +777,24 @@ func (t *TableRestore) importEngine( return nil } -func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint, indexEngine *kv.ClosedEngine) error { +func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, cp *TableCheckpoint, + indexEngineCp *EngineCheckpoint, indexEngine *kv.ClosedEngine) error { if cp.Status < CheckpointStatusIndexImported { - // the lock ensures the import() step will not be concurrent. - rc.postProcessLock.Lock() - err := t.importKV(ctx, indexEngine) - rc.postProcessLock.Unlock() - rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported) - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusIndexImported) + var err error + if indexEngineCp.Status < CheckpointStatusImported { + // the lock ensures the import() step will not be concurrent. + rc.postProcessLock.Lock() + err = t.importKV(ctx, indexEngine) + rc.postProcessLock.Unlock() + rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported) + } + + // gofail: var FailBeforeIndexEngineImported int + // if FailBeforeIndexEngineImported > 0 { + // panic("forcing failure due to FailBeforeIndexEngineImported") + // } + + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusIndexImported) if err != nil { common.AppLogger.Errorf("[%[1]s] failed to import index engine: %v", t.tableName, err.Error()) return errors.Trace(err) @@ -805,7 +808,7 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c rc.alterTableLock.Lock() err := t.restoreTableMeta(ctx, rc.tidbMgr.db) rc.alterTableLock.Unlock() - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusAlteredAutoInc) + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusAlteredAutoInc) if err != nil { common.AppLogger.Errorf( "[%[1]s] failed to AUTO TABLE %[1]s SET AUTO_INCREMENT=%[2]d : %[3]v", @@ -819,10 +822,10 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c if cp.Status < CheckpointStatusChecksummed { if !rc.cfg.PostRestore.Checksum { common.AppLogger.Infof("[%s] Skip checksum.", t.tableName) - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, nil, CheckpointStatusChecksumSkipped) + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, nil, CheckpointStatusChecksumSkipped) } else { err := t.compareChecksum(ctx, rc.tidbMgr.db, cp) - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusChecksummed) + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusChecksummed) if err != nil { common.AppLogger.Errorf("[%s] checksum failed: %v", t.tableName, err.Error()) return errors.Trace(err) @@ -834,10 +837,10 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c if cp.Status < CheckpointStatusAnalyzed { if !rc.cfg.PostRestore.Analyze { common.AppLogger.Infof("[%s] Skip analyze.", t.tableName) - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, nil, CheckpointStatusAnalyzeSkipped) + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped) } else { err := t.analyzeTable(ctx, rc.tidbMgr.db) - rc.saveStatusCheckpoint(t.tableName, invalidEngineID, err, CheckpointStatusAnalyzed) + rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusAnalyzed) if err != nil { common.AppLogger.Errorf("[%s] analyze failed: %v", t.tableName, err.Error()) return errors.Trace(err) @@ -1094,11 +1097,14 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e } for _, chunk := range chunks { - for chunk.EngineID >= len(cp.Engines) { - cp.Engines = append(cp.Engines, &EngineCheckpoint{Status: CheckpointStatusLoaded}) + engine, found := cp.Engines[chunk.EngineID] + if !found { + engine = &EngineCheckpoint{ + EngineID: chunk.EngineID, + } + cp.Engines[chunk.EngineID] = engine } - cp.Engines[chunk.EngineID].EngineID = chunk.EngineID - cp.Engines[chunk.EngineID].Chunks = append(cp.Engines[chunk.EngineID].Chunks, &ChunkCheckpoint{ + engine.Chunks = append(engine.Chunks, &ChunkCheckpoint{ Key: ChunkCheckpointKey{ Path: chunk.File, Offset: chunk.Chunk.Offset, @@ -1109,7 +1115,7 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e } // Add index engine checkpoint - cp.Engines = append(cp.Engines, &EngineCheckpoint{EngineID: indexEngineID, Status: CheckpointStatusLoaded}) + cp.Engines[indexEngineID] = &EngineCheckpoint{EngineID: indexEngineID, Status: CheckpointStatusLoaded} common.AppLogger.Infof("[%s] load %d engines and %d chunks takes %v", t.tableName, len(cp.Engines), len(chunks), time.Since(timer)) return nil diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index c77d007e7..5e9952fb8 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -57,12 +57,13 @@ PARTIAL_IMPORT_QUERY="$PARTIAL_IMPORT_QUERY AS s;" # Set the failpoint to kill the lightning instance as soon as one table is imported # If checkpoint does work, this should only kill 9 instances of lightnings. -export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)' +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return(1)' # Start importing the tables. run_sql 'DROP DATABASE IF EXISTS cppk_tsr' run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' +# panic after saving index engine checkpoint status before saving table checkpoint status set +e for i in $(seq "$TABLE_COUNT"); do echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" @@ -71,6 +72,28 @@ for i in $(seq "$TABLE_COUNT"); do done set -e +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500)' +set +e +for i in $(seq "$TABLE_COUNT"); do + echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" + run_lightning 2> /dev/null +done +set -e + +# Start importing the tables. +run_sql 'DROP DATABASE IF EXISTS cppk_tsr' +run_sql 'DROP DATABASE IF EXISTS tidb_lightning_checkpoint_test_cppk' + +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailIfIndexEngineImported=return(1)' + +set +e +for i in $(seq "$TABLE_COUNT"); do + echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" + run_lightning #2> /dev/null + [ $? -ne 0 ] || exit 1 +done +set -e + # After everything is done, there should be no longer new calls to ImportEngine # (and thus `kill_lightning_after_one_import` will spare this final check) echo "******** Verify checkpoint no-op ********" From 94c3a88c8308b70a4e10ad5a569fe90dad3a6e9b Mon Sep 17 00:00:00 2001 From: Lonng Date: Wed, 6 Mar 2019 18:13:28 +0800 Subject: [PATCH 10/11] address comment --- lightning/config/config.go | 14 +- lightning/restore/checkpoints.go | 29 ++-- lightning/restore/file_checkpoints.pb.go | 196 ++++++----------------- lightning/restore/file_checkpoints.proto | 1 - lightning/restore/restore.go | 24 +-- tests/checkpoint/run.sh | 2 +- 6 files changed, 73 insertions(+), 193 deletions(-) diff --git a/lightning/config/config.go b/lightning/config/config.go index ea7067db5..0f52030f3 100644 --- a/lightning/config/config.go +++ b/lightning/config/config.go @@ -92,10 +92,10 @@ type Lightning struct { // PostRestore has some options which will be executed after kv restored. type PostRestore struct { - Level1Compact *bool `toml:"level-1-compact" json:"level-1-compact"` - Compact bool `toml:"compact" json:"compact"` - Checksum bool `toml:"checksum" json:"checksum"` - Analyze bool `toml:"analyze" json:"analyze"` + Level1Compact bool `toml:"level-1-compact" json:"level-1-compact"` + Compact bool `toml:"compact" json:"compact"` + Checksum bool `toml:"checksum" json:"checksum"` + Analyze bool `toml:"analyze" json:"analyze"` } type MydumperRuntime struct { @@ -230,11 +230,5 @@ func (cfg *Config) Load() error { cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb" } } - - // If the level 1 compact configuration not found, default to false - if cfg.PostRestore.Level1Compact == nil { - cfg.PostRestore.Level1Compact = new(bool) - *cfg.PostRestore.Level1Compact = false - } return nil } diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index c8088970e..a18e4d2c6 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -109,9 +109,8 @@ type ChunkCheckpoint struct { } type EngineCheckpoint struct { - EngineID int - Status CheckpointStatus - Chunks []*ChunkCheckpoint // a sorted array + Status CheckpointStatus + Chunks []*ChunkCheckpoint // a sorted array } type TableCheckpoint struct { @@ -438,8 +437,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab return errors.Trace(err) } cp.Engines[engineID] = &EngineCheckpoint{ - EngineID: engineID, - Status: CheckpointStatus(status), + Status: CheckpointStatus(status), } } if err := engineRows.Err(); err != nil { @@ -532,14 +530,14 @@ func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tab } defer chunkStmt.Close() - for _, engine := range checkpoints { - _, err = engineStmt.ExecContext(c, tableName, engine.EngineID, engine.Status) + for engineID, engine := range checkpoints { + _, err = engineStmt.ExecContext(c, tableName, engineID, engine.Status) if err != nil { return errors.Trace(err) } for _, value := range engine.Chunks { _, err = chunkStmt.ExecContext( - c, tableName, engine.EngineID, + c, tableName, engineID, value.Key.Path, value.Key.Offset, value.Columns, value.ShouldIncludeRowID, value.Chunk.Offset, value.Chunk.EndOffset, value.Chunk.PrevRowIDMax, value.Chunk.RowIDMax, value.Checksum.SumSize(), value.Checksum.SumKVS(), value.Checksum.Sum(), @@ -709,11 +707,10 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC Engines: make(map[int]*EngineCheckpoint, len(tableModel.Engines)), } - for _, engineModel := range tableModel.Engines { + for engineID, engineModel := range tableModel.Engines { engine := &EngineCheckpoint{ - EngineID: int(engineModel.EngineId), - Status: CheckpointStatus(engineModel.Status), - Chunks: make([]*ChunkCheckpoint, 0, len(engineModel.Chunks)), + Status: CheckpointStatus(engineModel.Status), + Chunks: make([]*ChunkCheckpoint, 0, len(engineModel.Chunks)), } for _, chunkModel := range engineModel.Chunks { @@ -738,7 +735,7 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC return engine.Chunks[i].Key.less(&engine.Chunks[j].Key) }) - cp.Engines[engine.EngineID] = engine + cp.Engines[int(engineID)] = engine } return cp, nil @@ -749,9 +746,9 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN defer cpdb.lock.Unlock() tableModel := cpdb.checkpoints.Checkpoints[tableName] - for _, engine := range checkpoints { + for engineID, engine := range checkpoints { engineModel := &EngineCheckpointModel{ - EngineId: int32(engine.EngineID), + EngineId: int32(engineID), Status: uint32(CheckpointStatusLoaded), Chunks: make(map[string]*ChunkCheckpointModel), } @@ -775,7 +772,7 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk.KvcKvs = value.Checksum.SumKVS() chunk.KvcChecksum = value.Checksum.Sum() } - tableModel.Engines[int32(engine.EngineID)] = engineModel + tableModel.Engines[int32(engineID)] = engineModel } return errors.Trace(cpdb.save()) diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index 8174f4423..b2aab30e6 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -62,11 +62,10 @@ func (m *CheckpointsModel) XXX_DiscardUnknown() { var xxx_messageInfo_CheckpointsModel proto.InternalMessageInfo type TableCheckpointModel struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` - Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` - AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` - Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - IndexEngineStatuses []uint32 `protobuf:"varint,7,rep,packed,name=index_engine_statuses,json=indexEngineStatuses,proto3" json:"index_engine_statuses,omitempty"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` + AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` + Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *TableCheckpointModel) Reset() { *m = TableCheckpointModel{} } @@ -204,46 +203,45 @@ func init() { } var fileDescriptor_c47ec4f2f281cd62 = []byte{ - // 617 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xcd, 0xc6, 0xcd, 0xd7, 0x24, 0x45, 0xd5, 0xd2, 0x94, 0x55, 0x10, 0x96, 0xa9, 0x40, 0xb2, - 0x04, 0x24, 0xa2, 0x5c, 0x50, 0xc5, 0xa9, 0xa5, 0x87, 0x0a, 0x55, 0xa0, 0x05, 0x2e, 0x5c, 0x2c, - 0xc7, 0xde, 0xc4, 0x96, 0x1d, 0x6f, 0xe4, 0xb5, 0xdd, 0xf6, 0x5f, 0xf0, 0x2f, 0x38, 0xf2, 0x37, - 0x7a, 0xec, 0x11, 0x21, 0x0e, 0xd0, 0xfe, 0x11, 0xb4, 0x1f, 0x55, 0x1d, 0x14, 0x55, 0xdc, 0x76, - 0xde, 0x7b, 0x33, 0xa3, 0xf7, 0x34, 0x36, 0xb8, 0x69, 0x3c, 0x8f, 0x8a, 0x2c, 0xce, 0xe6, 0x93, - 0x9c, 0x89, 0x82, 0xe7, 0x6c, 0x32, 0x8b, 0x53, 0xe6, 0x05, 0x11, 0x0b, 0x92, 0x25, 0x8f, 0xb3, - 0x42, 0x8c, 0x97, 0x39, 0x2f, 0xf8, 0xe8, 0xc5, 0x3c, 0x2e, 0xa2, 0x72, 0x3a, 0x0e, 0xf8, 0x62, - 0x32, 0xe7, 0x73, 0x3e, 0x51, 0xf0, 0xb4, 0x9c, 0xa9, 0x4a, 0x15, 0xea, 0xa5, 0xe5, 0xbb, 0xdf, - 0x11, 0x6c, 0x1d, 0xde, 0x0e, 0x39, 0xe1, 0x21, 0x4b, 0xf1, 0x5b, 0xe8, 0xd7, 0x06, 0x13, 0xe4, - 0x58, 0x6e, 0x7f, 0x6f, 0x77, 0xfc, 0xaf, 0xae, 0x0e, 0x1c, 0x65, 0x45, 0x7e, 0x4e, 0xeb, 0x6d, - 0xa3, 0xcf, 0x2b, 0x93, 0x95, 0x00, 0x6f, 0x81, 0x95, 0xb0, 0x73, 0x82, 0x1c, 0xe4, 0xf6, 0xa8, - 0x7c, 0xe2, 0x67, 0xd0, 0xaa, 0xfc, 0xb4, 0x64, 0xa4, 0xe9, 0x20, 0xb7, 0xbf, 0x37, 0x1c, 0x7f, - 0xf2, 0xa7, 0x29, 0xbb, 0x6d, 0x54, 0x9b, 0xa8, 0xd6, 0xec, 0x37, 0x5f, 0xa3, 0xdd, 0x6f, 0x4d, - 0xd8, 0x5e, 0xa7, 0xc1, 0x18, 0x36, 0x22, 0x5f, 0x44, 0x6a, 0xf8, 0x80, 0xaa, 0x37, 0xde, 0x81, - 0xb6, 0x28, 0xfc, 0xa2, 0x14, 0xc4, 0x72, 0x90, 0xbb, 0x49, 0x4d, 0x85, 0x1f, 0x01, 0xf8, 0x69, - 0xca, 0x03, 0x6f, 0xea, 0x0b, 0x46, 0x36, 0x1c, 0xe4, 0x5a, 0xb4, 0xa7, 0x90, 0x03, 0x5f, 0x30, - 0xfc, 0x06, 0x3a, 0x2c, 0x9b, 0xc7, 0x19, 0x13, 0xa4, 0x6b, 0xcc, 0xaf, 0x5b, 0x39, 0x3e, 0xd2, - 0x22, 0x6d, 0xfe, 0xa6, 0x05, 0xef, 0xc1, 0x30, 0xce, 0x42, 0x76, 0xe6, 0x69, 0xc0, 0xd3, 0x3b, - 0x99, 0x20, 0x1d, 0xc7, 0x72, 0x37, 0xe9, 0x7d, 0x45, 0xea, 0xd6, 0x8f, 0x86, 0x1a, 0x51, 0x18, - 0xd4, 0x87, 0xd5, 0x83, 0x6a, 0xe9, 0xa0, 0x9e, 0xaf, 0x06, 0xb5, 0x63, 0x96, 0xdf, 0x91, 0xd4, - 0x4f, 0x04, 0xc3, 0xb5, 0xa2, 0x5a, 0x2c, 0x68, 0x25, 0x96, 0x7d, 0x68, 0x07, 0x51, 0x99, 0x25, - 0x82, 0x34, 0x8d, 0xed, 0xb5, 0xfd, 0xe3, 0x43, 0x25, 0xd2, 0xb6, 0x4d, 0x07, 0x7e, 0x08, 0x3d, - 0xe3, 0x37, 0x0e, 0x55, 0xda, 0x2d, 0xda, 0xd5, 0xc0, 0x71, 0x38, 0xfa, 0x00, 0xfd, 0x5a, 0xcf, - 0xff, 0x9c, 0x81, 0x92, 0xdf, 0x61, 0xee, 0x57, 0x13, 0xb6, 0xd7, 0x69, 0xe4, 0x19, 0x2c, 0xfd, - 0x22, 0x32, 0xc3, 0xd5, 0x5b, 0xfa, 0xe5, 0xb3, 0x99, 0x60, 0x85, 0x1a, 0x6f, 0x51, 0x53, 0x61, - 0x02, 0x9d, 0x80, 0xa7, 0xe5, 0x22, 0xd3, 0xf7, 0x31, 0xa0, 0x37, 0x25, 0x7e, 0x09, 0x43, 0x11, - 0xf1, 0x32, 0x0d, 0xbd, 0x38, 0x0b, 0xd2, 0x32, 0x64, 0x5e, 0xce, 0x4f, 0xa5, 0x33, 0x79, 0x2b, - 0x5d, 0x8a, 0x35, 0x79, 0xac, 0x39, 0xca, 0x4f, 0x8f, 0x43, 0x79, 0x53, 0x2c, 0x0b, 0x3d, 0xb3, - 0xa8, 0xa5, 0x6f, 0x8a, 0x65, 0xe1, 0x7b, 0xbd, 0x6b, 0x0b, 0xac, 0x25, 0x17, 0xa4, 0xad, 0x70, - 0xf9, 0xc4, 0x4f, 0xe0, 0xde, 0x32, 0x67, 0x95, 0x9c, 0x1c, 0x87, 0xde, 0xc2, 0x3f, 0x23, 0x1d, - 0x45, 0x0e, 0x24, 0x4a, 0x25, 0x78, 0xe2, 0x9f, 0xc9, 0x5c, 0x6f, 0x05, 0x5d, 0x25, 0xe8, 0xe6, - 0x35, 0x32, 0xa9, 0x02, 0x6f, 0x7a, 0x5e, 0x30, 0x41, 0x7a, 0x0e, 0x72, 0x37, 0x68, 0x37, 0xa9, - 0x82, 0x03, 0x59, 0xe3, 0x07, 0xd0, 0x91, 0x64, 0x52, 0x09, 0x02, 0x8a, 0x6a, 0x27, 0x55, 0xf0, - 0xae, 0x12, 0xf8, 0x31, 0x0c, 0x24, 0xa1, 0x3e, 0x56, 0x51, 0x2e, 0x48, 0xdf, 0x41, 0x6e, 0x9b, - 0xf6, 0x93, 0x2a, 0x38, 0x34, 0xd0, 0xc1, 0xd3, 0x8b, 0x3f, 0x76, 0xe3, 0xe2, 0xca, 0x46, 0x97, - 0x57, 0x36, 0xfa, 0x7d, 0x65, 0xa3, 0xaf, 0xd7, 0x76, 0xe3, 0xf2, 0xda, 0x6e, 0xfc, 0xb8, 0xb6, - 0x1b, 0x5f, 0x3a, 0xe6, 0x27, 0x34, 0x6d, 0xab, 0xbf, 0xc8, 0xab, 0xbf, 0x01, 0x00, 0x00, 0xff, - 0xff, 0x6e, 0x1a, 0x8d, 0x1b, 0xa0, 0x04, 0x00, 0x00, + // 593 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xcd, 0x24, 0x4d, 0xe2, 0x5c, 0xe7, 0xfb, 0x54, 0x8d, 0xda, 0x32, 0x0a, 0xc2, 0x32, 0x15, + 0x48, 0x96, 0x00, 0x47, 0x94, 0x0d, 0xaa, 0x58, 0xb5, 0x74, 0x51, 0xa1, 0x0a, 0x34, 0x82, 0x0d, + 0x1b, 0xcb, 0x3f, 0x13, 0xdb, 0xb2, 0xe3, 0x89, 0x3c, 0xb6, 0xdb, 0xbe, 0x05, 0x6f, 0xc2, 0x6b, + 0x74, 0xd9, 0x25, 0x42, 0x2c, 0xa0, 0x7d, 0x04, 0x5e, 0x00, 0xcd, 0x8c, 0xab, 0xba, 0x28, 0xaa, + 0xd8, 0xdd, 0x7b, 0xce, 0xb9, 0xe7, 0xea, 0xf8, 0xda, 0x06, 0x27, 0x4f, 0xe3, 0xa4, 0x2a, 0xd2, + 0x22, 0x9e, 0x97, 0x4c, 0x54, 0xbc, 0x64, 0xf3, 0x45, 0x9a, 0x33, 0x2f, 0x4c, 0x58, 0x98, 0xad, + 0x78, 0x5a, 0x54, 0xc2, 0x5d, 0x95, 0xbc, 0xe2, 0xb3, 0x17, 0x71, 0x5a, 0x25, 0x75, 0xe0, 0x86, + 0x7c, 0x39, 0x8f, 0x79, 0xcc, 0xe7, 0x0a, 0x0e, 0xea, 0x85, 0xea, 0x54, 0xa3, 0x2a, 0x2d, 0xdf, + 0xfd, 0x8a, 0x60, 0xf3, 0xf0, 0xd6, 0xe4, 0x84, 0x47, 0x2c, 0xc7, 0x6f, 0xc1, 0xec, 0x18, 0x13, + 0x64, 0x0f, 0x1c, 0x73, 0x6f, 0xd7, 0xfd, 0x5b, 0xd7, 0x05, 0x8e, 0x8a, 0xaa, 0x3c, 0xa7, 0xdd, + 0xb1, 0xd9, 0xa7, 0x3b, 0xce, 0x4a, 0x80, 0x37, 0x61, 0x90, 0xb1, 0x73, 0x82, 0x6c, 0xe4, 0x4c, + 0xa8, 0x2c, 0xf1, 0x33, 0x18, 0x36, 0x7e, 0x5e, 0x33, 0xd2, 0xb7, 0x91, 0x63, 0xee, 0x6d, 0xbb, + 0x1f, 0xfd, 0x20, 0x67, 0xb7, 0x83, 0x6a, 0x13, 0xd5, 0x9a, 0xfd, 0xfe, 0x6b, 0xb4, 0xfb, 0x1b, + 0xc1, 0xd6, 0x3a, 0x0d, 0xc6, 0xb0, 0x91, 0xf8, 0x22, 0x51, 0xe6, 0x53, 0xaa, 0x6a, 0xbc, 0x03, + 0x23, 0x51, 0xf9, 0x55, 0x2d, 0xc8, 0xc0, 0x46, 0xce, 0x7f, 0xb4, 0xed, 0xf0, 0x23, 0x00, 0x3f, + 0xcf, 0x79, 0xe8, 0x05, 0xbe, 0x60, 0x64, 0xc3, 0x46, 0xce, 0x80, 0x4e, 0x14, 0x72, 0xe0, 0x0b, + 0x86, 0xdf, 0xc0, 0x98, 0x15, 0x71, 0x5a, 0x30, 0x41, 0x8c, 0x36, 0xfc, 0xba, 0x95, 0xee, 0x91, + 0x16, 0xe9, 0xf0, 0x37, 0x23, 0x33, 0x0a, 0xd3, 0x2e, 0xd1, 0x0d, 0x3d, 0xd4, 0xa1, 0x9f, 0xdf, + 0x0d, 0xbd, 0xd3, 0x1a, 0xdd, 0x93, 0xfa, 0x3b, 0x82, 0xed, 0xb5, 0xa2, 0x4e, 0x44, 0x74, 0x27, + 0xe2, 0x3e, 0x8c, 0xc2, 0xa4, 0x2e, 0x32, 0x41, 0xfa, 0x6d, 0x84, 0xb5, 0xf3, 0xee, 0xa1, 0x12, + 0xe9, 0x08, 0xed, 0x04, 0x7e, 0x08, 0x13, 0x1d, 0xc6, 0x4b, 0x23, 0xf5, 0xe4, 0x86, 0xd4, 0xd0, + 0xc0, 0x71, 0x34, 0xfb, 0x00, 0x66, 0x67, 0xe6, 0x5f, 0x4e, 0xaa, 0xe4, 0xf7, 0x84, 0xfb, 0xd1, + 0x87, 0xad, 0x75, 0x1a, 0x79, 0xd2, 0x95, 0x5f, 0x25, 0xad, 0xb9, 0xaa, 0x65, 0x5e, 0xbe, 0x58, + 0x08, 0x56, 0x29, 0xfb, 0x01, 0x6d, 0x3b, 0x4c, 0x60, 0x1c, 0xf2, 0xbc, 0x5e, 0x16, 0xfa, 0xd6, + 0x53, 0x7a, 0xd3, 0xe2, 0x97, 0xb0, 0x2d, 0x12, 0x5e, 0xe7, 0x91, 0x97, 0x16, 0x61, 0x5e, 0x47, + 0xcc, 0x2b, 0xf9, 0xa9, 0x4c, 0x26, 0xef, 0x6e, 0x50, 0xac, 0xc9, 0x63, 0xcd, 0x51, 0x7e, 0x7a, + 0x1c, 0xc9, 0xf7, 0x83, 0x15, 0x91, 0xd7, 0x2e, 0x1a, 0xea, 0xf7, 0x83, 0x15, 0xd1, 0x7b, 0xbd, + 0x6b, 0x13, 0x06, 0x2b, 0x2e, 0xc8, 0x48, 0xe1, 0xb2, 0xc4, 0x4f, 0xe0, 0xff, 0x55, 0xc9, 0x1a, + 0xe9, 0x9c, 0x46, 0xde, 0xd2, 0x3f, 0x23, 0x63, 0x45, 0x4e, 0x25, 0x4a, 0x25, 0x78, 0xe2, 0x9f, + 0xc9, 0xe7, 0x7a, 0x2b, 0x30, 0x94, 0xc0, 0x28, 0x3b, 0x64, 0xd6, 0x84, 0x5e, 0x70, 0x5e, 0x31, + 0x41, 0x26, 0x36, 0x72, 0x36, 0xa8, 0x91, 0x35, 0xe1, 0x81, 0xec, 0xf1, 0x03, 0x18, 0x4b, 0x32, + 0x6b, 0x04, 0x01, 0x45, 0x8d, 0xb2, 0x26, 0x7c, 0xd7, 0x08, 0xfc, 0x18, 0xa6, 0x92, 0x50, 0x1f, + 0x9e, 0xa8, 0x97, 0xc4, 0xb4, 0x91, 0x33, 0xa2, 0x66, 0xd6, 0x84, 0x87, 0x2d, 0x74, 0xf0, 0xf4, + 0xe2, 0x97, 0xd5, 0xbb, 0xb8, 0xb2, 0xd0, 0xe5, 0x95, 0x85, 0x7e, 0x5e, 0x59, 0xe8, 0xcb, 0xb5, + 0xd5, 0xbb, 0xbc, 0xb6, 0x7a, 0xdf, 0xae, 0xad, 0xde, 0xe7, 0x71, 0xfb, 0x43, 0x09, 0x46, 0xea, + 0x8f, 0xf0, 0xea, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xac, 0xc5, 0x94, 0x1b, 0x6c, 0x04, 0x00, + 0x00, } func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { @@ -323,23 +321,6 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(m.AllocBase)) } - if len(m.IndexEngineStatuses) > 0 { - dAtA3 := make([]byte, len(m.IndexEngineStatuses)*10) - var j2 int - for _, num := range m.IndexEngineStatuses { - for num >= 1<<7 { - dAtA3[j2] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j2++ - } - dAtA3[j2] = uint8(num) - j2++ - } - dAtA[i] = 0x3a - i++ - i = encodeVarintFileCheckpoints(dAtA, i, uint64(j2)) - i += copy(dAtA[i:], dAtA3[:j2]) - } if len(m.Engines) > 0 { for k, _ := range m.Engines { dAtA[i] = 0x42 @@ -359,11 +340,11 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(v.Size())) - n4, err := v.MarshalTo(dAtA[i:]) + n2, err := v.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n4 + i += n2 } } } @@ -410,11 +391,11 @@ func (m *EngineCheckpointModel) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintFileCheckpoints(dAtA, i, uint64(v.Size())) - n5, err := v.MarshalTo(dAtA[i:]) + n3, err := v.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n3 } } } @@ -554,13 +535,6 @@ func (m *TableCheckpointModel) Size() (n int) { if m.AllocBase != 0 { n += 1 + sovFileCheckpoints(uint64(m.AllocBase)) } - if len(m.IndexEngineStatuses) > 0 { - l = 0 - for _, e := range m.IndexEngineStatuses { - l += sovFileCheckpoints(uint64(e)) - } - n += 1 + sovFileCheckpoints(uint64(l)) + l - } if len(m.Engines) > 0 { for k, v := range m.Engines { _ = k @@ -945,82 +919,6 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { break } } - case 7: - if wireType == 0 { - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.IndexEngineStatuses = append(m.IndexEngineStatuses, v) - } else if wireType == 2 { - var packedLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - packedLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if packedLen < 0 { - return ErrInvalidLengthFileCheckpoints - } - postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthFileCheckpoints - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - var elementCount int - var count int - for _, integer := range dAtA[iNdEx:postIndex] { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.IndexEngineStatuses) == 0 { - m.IndexEngineStatuses = make([]uint32, 0, elementCount) - } - for iNdEx < postIndex { - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.IndexEngineStatuses = append(m.IndexEngineStatuses, v) - } - } else { - return fmt.Errorf("proto: wrong wireType = %d for field IndexEngineStatuses", wireType) - } case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Engines", wireType) diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index 183bf475f..0c06407f6 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -28,7 +28,6 @@ message TableCheckpointModel { uint32 status = 3; int64 alloc_base = 4; map engines = 8; - repeated uint32 index_engine_statuses = 7; } message EngineCheckpointModel { diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 640095c0c..388ed83fa 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -58,7 +58,7 @@ const ( const ( indexEngineID = -1 - wholeTableEngineID = math.MinInt64 + wholeTableEngineID = math.MaxInt64 ) const ( @@ -523,13 +523,7 @@ func (t *TableRestore) restoreTable( } // 2. Restore engines (if still needed) - var indexEngineCp *EngineCheckpoint - for _, checkpoint := range cp.Engines { - if checkpoint.EngineID == indexEngineID { - indexEngineCp = checkpoint - break - } - } + indexEngineCp := cp.Engines[indexEngineID] if indexEngineCp == nil { return fmt.Errorf("table %v index engine checkpoint not found", t.tableName) } @@ -564,7 +558,7 @@ func (t *TableRestore) restoreTable( var wg sync.WaitGroup var engineErr common.OnceError - for _, engine := range cp.Engines { + for engineID, engine := range cp.Engines { select { case <-ctx.Done(): return ctx.Err() @@ -575,7 +569,7 @@ func (t *TableRestore) restoreTable( } // Should skip index engine - if engine.EngineID < 0 { + if engineID < 0 { continue } @@ -601,7 +595,7 @@ func (t *TableRestore) restoreTable( if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { engineErr.Set(tag, err) } - }(restoreWorker, engine.EngineID, engine) + }(restoreWorker, engineID, engine) } wg.Wait() @@ -762,7 +756,7 @@ func (t *TableRestore) importEngine( } // 2. perform a level-1 compact if idling. - if *rc.cfg.PostRestore.Level1Compact && + if rc.cfg.PostRestore.Level1Compact && atomic.CompareAndSwapInt32(&rc.compactState, compactStateIdle, compactStateDoing) { go func() { err := rc.doCompact(ctx, Level1Compact) @@ -1099,9 +1093,7 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e for _, chunk := range chunks { engine, found := cp.Engines[chunk.EngineID] if !found { - engine = &EngineCheckpoint{ - EngineID: chunk.EngineID, - } + engine = &EngineCheckpoint{} cp.Engines[chunk.EngineID] = engine } engine.Chunks = append(engine.Chunks, &ChunkCheckpoint{ @@ -1115,7 +1107,7 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e } // Add index engine checkpoint - cp.Engines[indexEngineID] = &EngineCheckpoint{EngineID: indexEngineID, Status: CheckpointStatusLoaded} + cp.Engines[indexEngineID] = &EngineCheckpoint{Status: CheckpointStatusLoaded} common.AppLogger.Infof("[%s] load %d engines and %d chunks takes %v", t.tableName, len(cp.Engines), len(chunks), time.Since(timer)) return nil diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index 5e9952fb8..e292966f9 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -89,7 +89,7 @@ export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/Sl set +e for i in $(seq "$TABLE_COUNT"); do echo "******** Importing Table Now (step $i/$TABLE_COUNT) ********" - run_lightning #2> /dev/null + run_lightning 2> /dev/null [ $? -ne 0 ] || exit 1 done set -e From 0488877c632a40543d6ff40ac8b20578ff8271dd Mon Sep 17 00:00:00 2001 From: Lonng Date: Fri, 8 Mar 2019 15:42:54 +0800 Subject: [PATCH 11/11] address comment --- cmd/tidb-lightning-ctl/main.go | 4 +- lightning/kv/importer.go | 6 +- lightning/mydump/region.go | 4 +- lightning/mydump/region_test.go | 16 +-- lightning/restore/checkpoints.go | 43 ++++---- lightning/restore/file_checkpoints.pb.go | 120 +++++++++-------------- lightning/restore/file_checkpoints.proto | 3 +- lightning/restore/restore.go | 26 ++--- tests/checkpoint/run.sh | 2 +- 9 files changed, 98 insertions(+), 126 deletions(-) diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 836f2695a..f86e5737b 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -176,8 +176,8 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tableName s for _, table := range targetTables { for engineID := 0; engineID < table.EnginesCount; engineID++ { - fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) - closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, engineID) + fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, int32(engineID)) + closedEngine, err := importer.UnsafeCloseEngine(ctx, table.TableName, int32(engineID)) if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while closing engine:", err) lastErr = err diff --git a/lightning/kv/importer.go b/lightning/kv/importer.go index 2c92e9221..02064cfac 100644 --- a/lightning/kv/importer.go +++ b/lightning/kv/importer.go @@ -155,7 +155,7 @@ func isIgnorableOpenCloseEngineError(err error) bool { return err == nil || strings.Contains(err.Error(), "FileExists") } -func makeTag(tableName string, engineID int) string { +func makeTag(tableName string, engineID int32) string { return fmt.Sprintf("%s:%d", tableName, engineID) } @@ -166,7 +166,7 @@ var engineNamespace = uuid.Must(uuid.FromString("d68d6abe-c59e-45d6-ade8-e2b0ceb func (importer *Importer) OpenEngine( ctx context.Context, tableName string, - engineID int, + engineID int32, ) (*OpenedEngine, error) { tag := makeTag(tableName, engineID) engineUUID := uuid.NewV5(engineNamespace, tag) @@ -312,7 +312,7 @@ func (engine *OpenedEngine) Close(ctx context.Context) (*ClosedEngine, error) { // (Open -> Write -> Close -> Import). This method should only be used when one // knows via other ways that the engine has already been opened, e.g. when // resuming from a checkpoint. -func (importer *Importer) UnsafeCloseEngine(ctx context.Context, tableName string, engineID int) (*ClosedEngine, error) { +func (importer *Importer) UnsafeCloseEngine(ctx context.Context, tableName string, engineID int32) (*ClosedEngine, error) { tag := makeTag(tableName, engineID) engineUUID := uuid.NewV5(engineNamespace, tag) return importer.unsafeCloseEngine(ctx, tag, engineUUID) diff --git a/lightning/mydump/region.go b/lightning/mydump/region.go index 29b202fdb..054d48ec4 100644 --- a/lightning/mydump/region.go +++ b/lightning/mydump/region.go @@ -21,7 +21,7 @@ import ( ) type TableRegion struct { - EngineID int + EngineID int32 DB string Table string @@ -77,7 +77,7 @@ func AllocateEngineIDs( return } - curEngineID := 0 + curEngineID := int32(0) curEngineSize := 0.0 curBatchSize := batchSize diff --git a/lightning/mydump/region_test.go b/lightning/mydump/region_test.go index 5f9208cd0..82412a99a 100644 --- a/lightning/mydump/region_test.go +++ b/lightning/mydump/region_test.go @@ -123,8 +123,8 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { filesRegions = append(filesRegions, new(TableRegion)) } - checkEngineSizes := func(what string, expected map[int]int) { - actual := make(map[int]int) + checkEngineSizes := func(what string, expected map[int32]int) { + actual := make(map[int32]int) for _, region := range filesRegions { actual[region.EngineID]++ } @@ -133,13 +133,13 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { // Batch size > Total size => Everything in the zero batch. AllocateEngineIDs(filesRegions, dataFileSizes, 1000, 0.5, 1000) - checkEngineSizes("no batching", map[int]int{ + checkEngineSizes("no batching", map[int32]int{ 0: 700, }) // Allocate 3 engines. AllocateEngineIDs(filesRegions, dataFileSizes, 200, 0.5, 1000) - checkEngineSizes("batch size = 200", map[int]int{ + checkEngineSizes("batch size = 200", map[int32]int{ 0: 170, 1: 213, 2: 317, @@ -147,7 +147,7 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { // Allocate 3 engines with an alternative ratio AllocateEngineIDs(filesRegions, dataFileSizes, 200, 0.6, 1000) - checkEngineSizes("batch size = 200, ratio = 0.6", map[int]int{ + checkEngineSizes("batch size = 200, ratio = 0.6", map[int32]int{ 0: 160, 1: 208, 2: 332, @@ -155,7 +155,7 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { // Allocate 5 engines. AllocateEngineIDs(filesRegions, dataFileSizes, 100, 0.5, 1000) - checkEngineSizes("batch size = 100", map[int]int{ + checkEngineSizes("batch size = 100", map[int32]int{ 0: 93, 1: 105, 2: 122, @@ -165,7 +165,7 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { // Number of engines > table concurrency AllocateEngineIDs(filesRegions, dataFileSizes, 50, 0.5, 4) - checkEngineSizes("batch size = 50, limit table conc = 4", map[int]int{ + checkEngineSizes("batch size = 50, limit table conc = 4", map[int32]int{ 0: 50, 1: 59, 2: 73, @@ -183,7 +183,7 @@ func (s *testMydumpRegionSuite) TestAllocateEngineIDs(c *C) { // Zero ratio = Uniform AllocateEngineIDs(filesRegions, dataFileSizes, 100, 0.0, 1000) - checkEngineSizes("batch size = 100, ratio = 0", map[int]int{ + checkEngineSizes("batch size = 100, ratio = 0", map[int32]int{ 0: 100, 1: 100, 2: 100, diff --git a/lightning/restore/checkpoints.go b/lightning/restore/checkpoints.go index a18e4d2c6..65af26e39 100644 --- a/lightning/restore/checkpoints.go +++ b/lightning/restore/checkpoints.go @@ -116,7 +116,7 @@ type EngineCheckpoint struct { type TableCheckpoint struct { Status CheckpointStatus AllocBase int64 - Engines map[int]*EngineCheckpoint + Engines map[int32]*EngineCheckpoint } func (cp *TableCheckpoint) CountChunks() int { @@ -144,16 +144,16 @@ type TableCheckpointDiff struct { hasRebase bool status CheckpointStatus allocBase int64 - engines map[int]engineCheckpointDiff + engines map[int32]engineCheckpointDiff } func NewTableCheckpointDiff() *TableCheckpointDiff { return &TableCheckpointDiff{ - engines: make(map[int]engineCheckpointDiff), + engines: make(map[int32]engineCheckpointDiff), } } -func (cpd *TableCheckpointDiff) insertEngineCheckpointDiff(engineID int, newDiff engineCheckpointDiff) { +func (cpd *TableCheckpointDiff) insertEngineCheckpointDiff(engineID int32, newDiff engineCheckpointDiff) { if oldDiff, ok := cpd.engines[engineID]; ok { if newDiff.hasStatus { oldDiff.hasStatus = true @@ -184,7 +184,7 @@ type TableCheckpointMerger interface { } type StatusCheckpointMerger struct { - EngineID int // wholeTableEngineID == apply to whole table. + EngineID int32 // wholeTableEngineID == apply to whole table. Status CheckpointStatus } @@ -197,7 +197,7 @@ func (merger *StatusCheckpointMerger) MergeInto(cpd *TableCheckpointDiff) { cpd.status = merger.Status cpd.hasStatus = true } - if merger.EngineID >= 0 { + if merger.EngineID >= 0 && merger.EngineID != wholeTableEngineID { cpd.insertEngineCheckpointDiff(merger.EngineID, engineCheckpointDiff{ hasStatus: true, status: merger.Status, @@ -207,7 +207,7 @@ func (merger *StatusCheckpointMerger) MergeInto(cpd *TableCheckpointDiff) { } type ChunkCheckpointMerger struct { - EngineID int + EngineID int32 Key ChunkCheckpointKey Checksum verify.KVChecksum Pos int64 @@ -244,7 +244,7 @@ type CheckpointsDB interface { Initialize(ctx context.Context, dbInfo map[string]*TidbDBInfo) error Get(ctx context.Context, tableName string) (*TableCheckpoint, error) Close() error - InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error + InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int32]*EngineCheckpoint) error Update(checkpointDiffs map[string]*TableCheckpointDiff) RemoveCheckpoint(ctx context.Context, tableName string) error @@ -272,11 +272,11 @@ func (*NullCheckpointsDB) Close() error { func (*NullCheckpointsDB) Get(_ context.Context, _ string) (*TableCheckpoint, error) { return &TableCheckpoint{ Status: CheckpointStatusLoaded, - Engines: map[int]*EngineCheckpoint{}, + Engines: map[int32]*EngineCheckpoint{}, }, nil } -func (*NullCheckpointsDB) InsertEngineCheckpoints(_ context.Context, _ string, _ map[int]*EngineCheckpoint) error { +func (*NullCheckpointsDB) InsertEngineCheckpoints(_ context.Context, _ string, _ map[int32]*EngineCheckpoint) error { return nil } @@ -413,7 +413,7 @@ func (cpdb *MySQLCheckpointsDB) Close() error { func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*TableCheckpoint, error) { cp := &TableCheckpoint{ - Engines: map[int]*EngineCheckpoint{}, + Engines: map[int32]*EngineCheckpoint{}, } purpose := "(read checkpoint " + tableName + ")" @@ -430,7 +430,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab defer engineRows.Close() for engineRows.Next() { var ( - engineID int + engineID int32 status uint8 ) if err := engineRows.Scan(&engineID, &status); err != nil { @@ -462,7 +462,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab for chunkRows.Next() { var ( value = new(ChunkCheckpoint) - engineID int + engineID int32 kvcBytes uint64 kvcKVs uint64 kvcChecksum uint64 @@ -502,7 +502,7 @@ func (cpdb *MySQLCheckpointsDB) Get(ctx context.Context, tableName string) (*Tab return cp, nil } -func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error { +func (cpdb *MySQLCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tableName string, checkpoints map[int32]*EngineCheckpoint) error { err := common.TransactWithRetry(ctx, cpdb.db, "(update engine checkpoints for "+tableName+")", func(c context.Context, tx *sql.Tx) error { engineStmt, err := tx.PrepareContext(c, fmt.Sprintf(` REPLACE INTO %s.%s (table_name, engine_id, status) VALUES (?, ?, ?); @@ -704,7 +704,7 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC cp := &TableCheckpoint{ Status: CheckpointStatus(tableModel.Status), AllocBase: tableModel.AllocBase, - Engines: make(map[int]*EngineCheckpoint, len(tableModel.Engines)), + Engines: make(map[int32]*EngineCheckpoint, len(tableModel.Engines)), } for engineID, engineModel := range tableModel.Engines { @@ -735,22 +735,21 @@ func (cpdb *FileCheckpointsDB) Get(_ context.Context, tableName string) (*TableC return engine.Chunks[i].Key.less(&engine.Chunks[j].Key) }) - cp.Engines[int(engineID)] = engine + cp.Engines[engineID] = engine } return cp, nil } -func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableName string, checkpoints map[int]*EngineCheckpoint) error { +func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableName string, checkpoints map[int32]*EngineCheckpoint) error { cpdb.lock.Lock() defer cpdb.lock.Unlock() tableModel := cpdb.checkpoints.Checkpoints[tableName] for engineID, engine := range checkpoints { engineModel := &EngineCheckpointModel{ - EngineId: int32(engineID), - Status: uint32(CheckpointStatusLoaded), - Chunks: make(map[string]*ChunkCheckpointModel), + Status: uint32(CheckpointStatusLoaded), + Chunks: make(map[string]*ChunkCheckpointModel), } for _, value := range engine.Chunks { key := value.Key.String() @@ -772,7 +771,7 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN chunk.KvcKvs = value.Checksum.SumKVS() chunk.KvcChecksum = value.Checksum.Sum() } - tableModel.Engines[int32(engineID)] = engineModel + tableModel.Engines[engineID] = engineModel } return errors.Trace(cpdb.save()) @@ -791,7 +790,7 @@ func (cpdb *FileCheckpointsDB) Update(checkpointDiffs map[string]*TableCheckpoin tableModel.AllocBase = cpd.allocBase } for engineID, engineDiff := range cpd.engines { - engineModel := tableModel.Engines[int32(engineID)] + engineModel := tableModel.Engines[engineID] if engineDiff.hasStatus { engineModel.Status = uint32(engineDiff.status) } diff --git a/lightning/restore/file_checkpoints.pb.go b/lightning/restore/file_checkpoints.pb.go index b2aab30e6..b3b38bd41 100644 --- a/lightning/restore/file_checkpoints.pb.go +++ b/lightning/restore/file_checkpoints.pb.go @@ -65,7 +65,7 @@ type TableCheckpointModel struct { Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` Status uint32 `protobuf:"varint,3,opt,name=status,proto3" json:"status,omitempty"` AllocBase int64 `protobuf:"varint,4,opt,name=alloc_base,json=allocBase,proto3" json:"alloc_base,omitempty"` - Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Engines map[int32]*EngineCheckpointModel `protobuf:"bytes,8,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"zigzag32,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *TableCheckpointModel) Reset() { *m = TableCheckpointModel{} } @@ -104,8 +104,7 @@ var xxx_messageInfo_TableCheckpointModel proto.InternalMessageInfo type EngineCheckpointModel struct { Status uint32 `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"` // key is "$path:$offset" - Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - EngineId int32 `protobuf:"varint,3,opt,name=engine_id,json=engineId,proto3" json:"engine_id,omitempty"` + Chunks map[string]*ChunkCheckpointModel `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *EngineCheckpointModel) Reset() { *m = EngineCheckpointModel{} } @@ -203,45 +202,44 @@ func init() { } var fileDescriptor_c47ec4f2f281cd62 = []byte{ - // 593 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xcd, 0x24, 0x4d, 0xe2, 0x5c, 0xe7, 0xfb, 0x54, 0x8d, 0xda, 0x32, 0x0a, 0xc2, 0x32, 0x15, - 0x48, 0x96, 0x00, 0x47, 0x94, 0x0d, 0xaa, 0x58, 0xb5, 0x74, 0x51, 0xa1, 0x0a, 0x34, 0x82, 0x0d, - 0x1b, 0xcb, 0x3f, 0x13, 0xdb, 0xb2, 0xe3, 0x89, 0x3c, 0xb6, 0xdb, 0xbe, 0x05, 0x6f, 0xc2, 0x6b, - 0x74, 0xd9, 0x25, 0x42, 0x2c, 0xa0, 0x7d, 0x04, 0x5e, 0x00, 0xcd, 0x8c, 0xab, 0xba, 0x28, 0xaa, - 0xd8, 0xdd, 0x7b, 0xce, 0xb9, 0xe7, 0xea, 0xf8, 0xda, 0x06, 0x27, 0x4f, 0xe3, 0xa4, 0x2a, 0xd2, - 0x22, 0x9e, 0x97, 0x4c, 0x54, 0xbc, 0x64, 0xf3, 0x45, 0x9a, 0x33, 0x2f, 0x4c, 0x58, 0x98, 0xad, - 0x78, 0x5a, 0x54, 0xc2, 0x5d, 0x95, 0xbc, 0xe2, 0xb3, 0x17, 0x71, 0x5a, 0x25, 0x75, 0xe0, 0x86, - 0x7c, 0x39, 0x8f, 0x79, 0xcc, 0xe7, 0x0a, 0x0e, 0xea, 0x85, 0xea, 0x54, 0xa3, 0x2a, 0x2d, 0xdf, - 0xfd, 0x8a, 0x60, 0xf3, 0xf0, 0xd6, 0xe4, 0x84, 0x47, 0x2c, 0xc7, 0x6f, 0xc1, 0xec, 0x18, 0x13, - 0x64, 0x0f, 0x1c, 0x73, 0x6f, 0xd7, 0xfd, 0x5b, 0xd7, 0x05, 0x8e, 0x8a, 0xaa, 0x3c, 0xa7, 0xdd, - 0xb1, 0xd9, 0xa7, 0x3b, 0xce, 0x4a, 0x80, 0x37, 0x61, 0x90, 0xb1, 0x73, 0x82, 0x6c, 0xe4, 0x4c, - 0xa8, 0x2c, 0xf1, 0x33, 0x18, 0x36, 0x7e, 0x5e, 0x33, 0xd2, 0xb7, 0x91, 0x63, 0xee, 0x6d, 0xbb, - 0x1f, 0xfd, 0x20, 0x67, 0xb7, 0x83, 0x6a, 0x13, 0xd5, 0x9a, 0xfd, 0xfe, 0x6b, 0xb4, 0xfb, 0x1b, - 0xc1, 0xd6, 0x3a, 0x0d, 0xc6, 0xb0, 0x91, 0xf8, 0x22, 0x51, 0xe6, 0x53, 0xaa, 0x6a, 0xbc, 0x03, - 0x23, 0x51, 0xf9, 0x55, 0x2d, 0xc8, 0xc0, 0x46, 0xce, 0x7f, 0xb4, 0xed, 0xf0, 0x23, 0x00, 0x3f, - 0xcf, 0x79, 0xe8, 0x05, 0xbe, 0x60, 0x64, 0xc3, 0x46, 0xce, 0x80, 0x4e, 0x14, 0x72, 0xe0, 0x0b, - 0x86, 0xdf, 0xc0, 0x98, 0x15, 0x71, 0x5a, 0x30, 0x41, 0x8c, 0x36, 0xfc, 0xba, 0x95, 0xee, 0x91, - 0x16, 0xe9, 0xf0, 0x37, 0x23, 0x33, 0x0a, 0xd3, 0x2e, 0xd1, 0x0d, 0x3d, 0xd4, 0xa1, 0x9f, 0xdf, - 0x0d, 0xbd, 0xd3, 0x1a, 0xdd, 0x93, 0xfa, 0x3b, 0x82, 0xed, 0xb5, 0xa2, 0x4e, 0x44, 0x74, 0x27, - 0xe2, 0x3e, 0x8c, 0xc2, 0xa4, 0x2e, 0x32, 0x41, 0xfa, 0x6d, 0x84, 0xb5, 0xf3, 0xee, 0xa1, 0x12, - 0xe9, 0x08, 0xed, 0x04, 0x7e, 0x08, 0x13, 0x1d, 0xc6, 0x4b, 0x23, 0xf5, 0xe4, 0x86, 0xd4, 0xd0, - 0xc0, 0x71, 0x34, 0xfb, 0x00, 0x66, 0x67, 0xe6, 0x5f, 0x4e, 0xaa, 0xe4, 0xf7, 0x84, 0xfb, 0xd1, - 0x87, 0xad, 0x75, 0x1a, 0x79, 0xd2, 0x95, 0x5f, 0x25, 0xad, 0xb9, 0xaa, 0x65, 0x5e, 0xbe, 0x58, - 0x08, 0x56, 0x29, 0xfb, 0x01, 0x6d, 0x3b, 0x4c, 0x60, 0x1c, 0xf2, 0xbc, 0x5e, 0x16, 0xfa, 0xd6, - 0x53, 0x7a, 0xd3, 0xe2, 0x97, 0xb0, 0x2d, 0x12, 0x5e, 0xe7, 0x91, 0x97, 0x16, 0x61, 0x5e, 0x47, - 0xcc, 0x2b, 0xf9, 0xa9, 0x4c, 0x26, 0xef, 0x6e, 0x50, 0xac, 0xc9, 0x63, 0xcd, 0x51, 0x7e, 0x7a, - 0x1c, 0xc9, 0xf7, 0x83, 0x15, 0x91, 0xd7, 0x2e, 0x1a, 0xea, 0xf7, 0x83, 0x15, 0xd1, 0x7b, 0xbd, - 0x6b, 0x13, 0x06, 0x2b, 0x2e, 0xc8, 0x48, 0xe1, 0xb2, 0xc4, 0x4f, 0xe0, 0xff, 0x55, 0xc9, 0x1a, - 0xe9, 0x9c, 0x46, 0xde, 0xd2, 0x3f, 0x23, 0x63, 0x45, 0x4e, 0x25, 0x4a, 0x25, 0x78, 0xe2, 0x9f, - 0xc9, 0xe7, 0x7a, 0x2b, 0x30, 0x94, 0xc0, 0x28, 0x3b, 0x64, 0xd6, 0x84, 0x5e, 0x70, 0x5e, 0x31, - 0x41, 0x26, 0x36, 0x72, 0x36, 0xa8, 0x91, 0x35, 0xe1, 0x81, 0xec, 0xf1, 0x03, 0x18, 0x4b, 0x32, - 0x6b, 0x04, 0x01, 0x45, 0x8d, 0xb2, 0x26, 0x7c, 0xd7, 0x08, 0xfc, 0x18, 0xa6, 0x92, 0x50, 0x1f, - 0x9e, 0xa8, 0x97, 0xc4, 0xb4, 0x91, 0x33, 0xa2, 0x66, 0xd6, 0x84, 0x87, 0x2d, 0x74, 0xf0, 0xf4, - 0xe2, 0x97, 0xd5, 0xbb, 0xb8, 0xb2, 0xd0, 0xe5, 0x95, 0x85, 0x7e, 0x5e, 0x59, 0xe8, 0xcb, 0xb5, - 0xd5, 0xbb, 0xbc, 0xb6, 0x7a, 0xdf, 0xae, 0xad, 0xde, 0xe7, 0x71, 0xfb, 0x43, 0x09, 0x46, 0xea, - 0x8f, 0xf0, 0xea, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xac, 0xc5, 0x94, 0x1b, 0x6c, 0x04, 0x00, - 0x00, + // 579 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0xce, 0xc6, 0xad, 0x93, 0x8c, 0x03, 0x2a, 0xab, 0xb6, 0xac, 0x82, 0xb0, 0x4c, 0x05, 0x92, + 0x25, 0xc0, 0x11, 0xe5, 0x82, 0x2a, 0x4e, 0x0d, 0x3d, 0x54, 0xa8, 0x02, 0xad, 0xe0, 0xc2, 0xc5, + 0xf2, 0xcf, 0xc6, 0xb6, 0xec, 0x78, 0x23, 0xaf, 0xed, 0x36, 0x6f, 0xc1, 0x9b, 0xf0, 0x04, 0xdc, + 0x7b, 0xec, 0x91, 0x03, 0x07, 0x48, 0x1e, 0x81, 0x17, 0x40, 0x5e, 0xbb, 0x8a, 0x83, 0xa2, 0x8a, + 0xdb, 0xcc, 0xf7, 0x7d, 0xf3, 0x4d, 0x3e, 0x8d, 0x37, 0x60, 0x26, 0x51, 0x10, 0xe6, 0x69, 0x94, + 0x06, 0xe3, 0x8c, 0x89, 0x9c, 0x67, 0x6c, 0x3c, 0x8d, 0x12, 0x66, 0x7b, 0x21, 0xf3, 0xe2, 0x39, + 0x8f, 0xd2, 0x5c, 0x58, 0xf3, 0x8c, 0xe7, 0x7c, 0xf4, 0x32, 0x88, 0xf2, 0xb0, 0x70, 0x2d, 0x8f, + 0xcf, 0xc6, 0x01, 0x0f, 0xf8, 0x58, 0xc2, 0x6e, 0x31, 0x95, 0x9d, 0x6c, 0x64, 0x55, 0xcb, 0x8f, + 0xbe, 0x21, 0xd8, 0x9b, 0xac, 0x4d, 0x2e, 0xb8, 0xcf, 0x12, 0xfc, 0x0e, 0xb4, 0x96, 0x31, 0x41, + 0x86, 0x62, 0x6a, 0xc7, 0x47, 0xd6, 0xbf, 0xba, 0x36, 0x70, 0x96, 0xe6, 0xd9, 0x82, 0xb6, 0xc7, + 0x46, 0x9f, 0x37, 0x9c, 0xa5, 0x00, 0xef, 0x81, 0x12, 0xb3, 0x05, 0x41, 0x06, 0x32, 0x07, 0xb4, + 0x2a, 0xf1, 0x73, 0xd8, 0x2d, 0x9d, 0xa4, 0x60, 0xa4, 0x6b, 0x20, 0x53, 0x3b, 0x3e, 0xb0, 0x3e, + 0x39, 0x6e, 0xc2, 0xd6, 0x83, 0x72, 0x13, 0xad, 0x35, 0x27, 0xdd, 0x37, 0xe8, 0xe8, 0x0f, 0x82, + 0xfd, 0x6d, 0x1a, 0x8c, 0x61, 0x27, 0x74, 0x44, 0x28, 0xcd, 0x87, 0x54, 0xd6, 0xf8, 0x10, 0x54, + 0x91, 0x3b, 0x79, 0x21, 0x88, 0x62, 0x20, 0xf3, 0x1e, 0x6d, 0x3a, 0xfc, 0x18, 0xc0, 0x49, 0x12, + 0xee, 0xd9, 0xae, 0x23, 0x18, 0xd9, 0x31, 0x90, 0xa9, 0xd0, 0x81, 0x44, 0x4e, 0x1d, 0xc1, 0xf0, + 0x5b, 0xe8, 0xb1, 0x34, 0x88, 0x52, 0x26, 0x48, 0xbf, 0x09, 0xbf, 0x6d, 0xa5, 0x75, 0x56, 0x8b, + 0xea, 0xf0, 0xb7, 0x23, 0x23, 0x0a, 0xc3, 0x36, 0xd1, 0x0e, 0xfd, 0xa0, 0x0e, 0xfd, 0x62, 0x33, + 0xf4, 0x61, 0x63, 0x74, 0x47, 0xea, 0xef, 0x08, 0x0e, 0xb6, 0x8a, 0x5a, 0x11, 0xd1, 0x46, 0xc4, + 0x13, 0x50, 0xbd, 0xb0, 0x48, 0x63, 0x41, 0xba, 0x4d, 0x84, 0xad, 0xf3, 0xd6, 0x44, 0x8a, 0xea, + 0x08, 0xcd, 0xc4, 0xe8, 0x23, 0x68, 0x2d, 0xf8, 0x7f, 0xae, 0x26, 0xe5, 0x77, 0xfc, 0xfe, 0x9f, + 0x5d, 0xd8, 0xdf, 0xa6, 0xa9, 0xae, 0x36, 0x77, 0xf2, 0xb0, 0x31, 0x97, 0x75, 0x15, 0x89, 0x4f, + 0xa7, 0x82, 0xe5, 0xd2, 0x5e, 0xa1, 0x4d, 0x87, 0x09, 0xf4, 0x3c, 0x9e, 0x14, 0xb3, 0xb4, 0x3e, + 0xe7, 0x90, 0xde, 0xb6, 0xf8, 0x15, 0x1c, 0x88, 0x90, 0x17, 0x89, 0x6f, 0x47, 0xa9, 0x97, 0x14, + 0x3e, 0xb3, 0x33, 0x7e, 0x69, 0x47, 0xbe, 0x3c, 0x6d, 0x9f, 0xe2, 0x9a, 0x3c, 0xaf, 0x39, 0xca, + 0x2f, 0xcf, 0xfd, 0xea, 0x13, 0x60, 0xa9, 0x6f, 0x37, 0x8b, 0x76, 0xeb, 0x4f, 0x80, 0xa5, 0xfe, + 0x87, 0x7a, 0xd7, 0x1e, 0x28, 0x73, 0x2e, 0x88, 0x2a, 0xf1, 0xaa, 0xc4, 0x4f, 0xe1, 0xfe, 0x3c, + 0x63, 0x65, 0xe5, 0x1c, 0xf9, 0xf6, 0xcc, 0xb9, 0x22, 0x3d, 0x49, 0x0e, 0x2b, 0x94, 0x56, 0xe0, + 0x85, 0x73, 0x85, 0x1f, 0xc1, 0x60, 0x2d, 0xe8, 0x4b, 0x41, 0x3f, 0x6b, 0x91, 0x71, 0xe9, 0xd9, + 0xee, 0x22, 0x67, 0x82, 0x0c, 0x0c, 0x64, 0xee, 0xd0, 0x7e, 0x5c, 0x7a, 0xa7, 0x55, 0x8f, 0x1f, + 0x42, 0xaf, 0x22, 0xe3, 0x52, 0x10, 0x90, 0x94, 0x1a, 0x97, 0xde, 0xfb, 0x52, 0xe0, 0x27, 0x30, + 0xac, 0x08, 0xf9, 0xb6, 0x44, 0x31, 0x23, 0x9a, 0x81, 0x4c, 0x95, 0x6a, 0x71, 0xe9, 0x4d, 0x1a, + 0xe8, 0xf4, 0xd9, 0xf5, 0x6f, 0xbd, 0x73, 0xbd, 0xd4, 0xd1, 0xcd, 0x52, 0x47, 0xbf, 0x96, 0x3a, + 0xfa, 0xba, 0xd2, 0x3b, 0x37, 0x2b, 0xbd, 0xf3, 0x63, 0xa5, 0x77, 0xbe, 0xf4, 0x9a, 0xff, 0x0c, + 0x57, 0x95, 0x8f, 0xfe, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd3, 0x90, 0x6a, 0x0b, 0x4f, + 0x04, 0x00, 0x00, } func (m *CheckpointsModel) Marshal() (dAtA []byte, err error) { @@ -331,11 +329,11 @@ func (m *TableCheckpointModel) MarshalTo(dAtA []byte) (int, error) { msgSize = v.Size() msgSize += 1 + sovFileCheckpoints(uint64(msgSize)) } - mapSize := 1 + sovFileCheckpoints(uint64(k)) + msgSize + mapSize := 1 + sozFileCheckpoints(uint64(k)) + msgSize i = encodeVarintFileCheckpoints(dAtA, i, uint64(mapSize)) dAtA[i] = 0x8 i++ - i = encodeVarintFileCheckpoints(dAtA, i, uint64(k)) + i = encodeVarintFileCheckpoints(dAtA, i, uint64((uint32(k)<<1)^uint32((k>>31)))) if v != nil { dAtA[i] = 0x12 i++ @@ -399,11 +397,6 @@ func (m *EngineCheckpointModel) MarshalTo(dAtA []byte) (int, error) { } } } - if m.EngineId != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintFileCheckpoints(dAtA, i, uint64(m.EngineId)) - } return i, nil } @@ -544,7 +537,7 @@ func (m *TableCheckpointModel) Size() (n int) { l = v.Size() l += 1 + sovFileCheckpoints(uint64(l)) } - mapEntrySize := 1 + sovFileCheckpoints(uint64(k)) + l + mapEntrySize := 1 + sozFileCheckpoints(uint64(k)) + l n += mapEntrySize + 1 + sovFileCheckpoints(uint64(mapEntrySize)) } } @@ -573,9 +566,6 @@ func (m *EngineCheckpointModel) Size() (n int) { n += mapEntrySize + 1 + sovFileCheckpoints(uint64(mapEntrySize)) } } - if m.EngineId != 0 { - n += 1 + sovFileCheckpoints(uint64(m.EngineId)) - } return n } @@ -972,6 +962,7 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } fieldNum := int32(wire >> 3) if fieldNum == 1 { + var mapkeytemp int32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowFileCheckpoints @@ -981,11 +972,13 @@ func (m *TableCheckpointModel) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - mapkey |= int32(b&0x7F) << shift + mapkeytemp |= int32(b&0x7F) << shift if b < 0x80 { break } } + mapkeytemp = int32((uint32(mapkeytemp) >> 1) ^ uint32(((mapkeytemp&1)<<31)>>31)) + mapkey = int32(mapkeytemp) } else if fieldNum == 2 { var mapmsglen int for shift := uint(0); ; shift += 7 { @@ -1235,25 +1228,6 @@ func (m *EngineCheckpointModel) Unmarshal(dAtA []byte) error { } m.Chunks[mapkey] = mapvalue iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EngineId", wireType) - } - m.EngineId = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowFileCheckpoints - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.EngineId |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipFileCheckpoints(dAtA[iNdEx:]) diff --git a/lightning/restore/file_checkpoints.proto b/lightning/restore/file_checkpoints.proto index 0c06407f6..3980e5ac0 100644 --- a/lightning/restore/file_checkpoints.proto +++ b/lightning/restore/file_checkpoints.proto @@ -27,14 +27,13 @@ message TableCheckpointModel { bytes hash = 1; uint32 status = 3; int64 alloc_base = 4; - map engines = 8; + map engines = 8; } message EngineCheckpointModel { uint32 status = 1; // key is "$path:$offset" map chunks = 2; - int32 engine_id = 3; } message ChunkCheckpointModel { diff --git a/lightning/restore/restore.go b/lightning/restore/restore.go index 388ed83fa..b43f62946 100644 --- a/lightning/restore/restore.go +++ b/lightning/restore/restore.go @@ -58,7 +58,7 @@ const ( const ( indexEngineID = -1 - wholeTableEngineID = math.MaxInt64 + wholeTableEngineID = math.MaxInt32 ) const ( @@ -294,7 +294,7 @@ func (rc *RestoreController) estimateChunkCountIntoMetrics() { metric.ChunkCounter.WithLabelValues(metric.ChunkStateEstimated).Add(float64(estimatedChunkCount)) } -func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int, err error, statusIfSucceed CheckpointStatus) { +func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int32, err error, statusIfSucceed CheckpointStatus) { merger := &StatusCheckpointMerger{Status: statusIfSucceed, EngineID: engineID} switch { @@ -525,7 +525,7 @@ func (t *TableRestore) restoreTable( // 2. Restore engines (if still needed) indexEngineCp := cp.Engines[indexEngineID] if indexEngineCp == nil { - return fmt.Errorf("table %v index engine checkpoint not found", t.tableName) + return errors.Errorf("table %v index engine checkpoint not found", t.tableName) } // The table checkpoint status set to `CheckpointStatusIndexImported` only if @@ -550,7 +550,7 @@ func (t *TableRestore) restoreTable( // that index engine checkpoint status less than `CheckpointStatusImported`. // So the index engine must be found in above process if indexEngine == nil { - return fmt.Errorf("table checkpoint status %v incompitable with index engine checkpoint status %v", + return errors.Errorf("table checkpoint status %v incompitable with index engine checkpoint status %v", cp.Status, indexEngineCp.Status) } @@ -580,7 +580,7 @@ func (t *TableRestore) restoreTable( // the difference between restoring tables concurrently and restoring tables one by one. restoreWorker := rc.tableWorkers.Apply() - go func(w *worker.Worker, eid int, ecp *EngineCheckpoint) { + go func(w *worker.Worker, eid int32, ecp *EngineCheckpoint) { defer wg.Done() tag := fmt.Sprintf("%s:%d", t.tableName, eid) @@ -615,7 +615,7 @@ func (t *TableRestore) restoreTable( rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusClosed) } if err != nil { - common.AppLogger.Errorf("[kv-deliver] index engine closed error: %s", errors.ErrorStack(err)) + common.AppLogger.Errorf("[%s] [kv-deliver] index engine closed error: %s", t.tableName, errors.ErrorStack(err)) return errors.Trace(err) } } @@ -628,7 +628,7 @@ func (t *TableRestore) restoreEngine( ctx context.Context, rc *RestoreController, indexEngine *kv.OpenedEngine, - engineID int, + engineID int32, cp *EngineCheckpoint, ) (*kv.ClosedEngine, *worker.Worker, error) { if cp.Status >= CheckpointStatusClosed { @@ -735,7 +735,7 @@ func (t *TableRestore) importEngine( ctx context.Context, closedEngine *kv.ClosedEngine, rc *RestoreController, - engineID int, + engineID int32, cp *EngineCheckpoint, ) error { if cp.Status >= CheckpointStatusImported { @@ -783,10 +783,8 @@ func (t *TableRestore) postProcess(ctx context.Context, rc *RestoreController, c rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusImported) } - // gofail: var FailBeforeIndexEngineImported int - // if FailBeforeIndexEngineImported > 0 { + // gofail: var FailBeforeIndexEngineImported struct{} // panic("forcing failure due to FailBeforeIndexEngineImported") - // } rc.saveStatusCheckpoint(t.tableName, wholeTableEngineID, err, CheckpointStatusIndexImported) if err != nil { @@ -1093,7 +1091,9 @@ func (t *TableRestore) populateChunks(cfg *config.Config, cp *TableCheckpoint) e for _, chunk := range chunks { engine, found := cp.Engines[chunk.EngineID] if !found { - engine = &EngineCheckpoint{} + engine = &EngineCheckpoint{ + Status: CheckpointStatusLoaded, + } cp.Engines[chunk.EngineID] = engine } engine.Chunks = append(engine.Chunks, &ChunkCheckpoint{ @@ -1333,7 +1333,7 @@ func splitIntoDeliveryStreams(totalKVs []kvenc.KvPair, splitSize int) [][]kvenc. func (cr *chunkRestore) restore( ctx context.Context, t *TableRestore, - engineID int, + engineID int32, dataEngine, indexEngine *kv.OpenedEngine, rc *RestoreController, ) error { diff --git a/tests/checkpoint/run.sh b/tests/checkpoint/run.sh index e292966f9..d018ba83a 100755 --- a/tests/checkpoint/run.sh +++ b/tests/checkpoint/run.sh @@ -57,7 +57,7 @@ PARTIAL_IMPORT_QUERY="$PARTIAL_IMPORT_QUERY AS s;" # Set the failpoint to kill the lightning instance as soon as one table is imported # If checkpoint does work, this should only kill 9 instances of lightnings. -export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return(1)' +export GOFAIL_FAILPOINTS='github.com/pingcap/tidb-lightning/lightning/restore/SlowDownImport=sleep(500);github.com/pingcap/tidb-lightning/lightning/restore/FailBeforeIndexEngineImported=return' # Start importing the tables. run_sql 'DROP DATABASE IF EXISTS cppk_tsr'