From 37cdfcdaa61311e3e5d96eea6de8b0e57842ff79 Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Sun, 21 Apr 2024 23:05:39 +0400 Subject: [PATCH 1/7] Fix pruning again --- eth/stagedsync/stage_log_index.go | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index d0c19faa1c9..2c6b5ac4584 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -395,11 +395,11 @@ func pruneOldLogChunks(tx kv.RwTx, bucket string, inMem *etl.Collector, pruneTo return err } var blockNum uint64 - if bucket == kv.Log { - blockNum = binary.BigEndian.Uint64(k) - } else { + // if bucket == kv.Log { + // blockNum = binary.BigEndian.Uint64(k) + // } else { blockNum = uint64(binary.BigEndian.Uint32(k[len(key):])) - } + // } if !bytes.HasPrefix(k, key) || blockNum >= pruneTo { break } @@ -438,7 +438,7 @@ func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, s.PruneProgress, pruneTo, ctx, logger, cfg.noPruneContracts); err != nil { return err } - if err = s.Done(tx); err != nil { + if err = s.DoneAt(tx, pruneTo); err != nil { return err } @@ -460,22 +460,25 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune defer topics.Close() addrs := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger) defer addrs.Close() - pruneLogKeyCollector := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger) - defer pruneLogKeyCollector.Close() + // pruneLogKeyCollector := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger) + // defer pruneLogKeyCollector.Close() reader := bytes.NewReader(nil) { - c, err := tx.Cursor(kv.Log) + c, err := tx.RwCursor(kv.Log) if err != nil { return err } defer c.Close() + log.Info("pruneLogIndex running", "pruneFrom", pruneFrom, "pruneTo", pruneTo) + for k, v, err := c.Seek(dbutils.LogKey(pruneFrom, 0)); k != nil; k, v, err = c.Next() { if err != nil { return err } blockNum := binary.BigEndian.Uint64(k) + log.Info("Pruneloop", "key", k, "blockNum", blockNum) if blockNum >= pruneTo { break } @@ -502,6 +505,8 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune break } } + log.Info("[SPIDERMAN] Pruneloop", "toPrune", toPrune) + if toPrune { for _, l := range logs { for _, topic := range l.Topics { @@ -513,9 +518,12 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune return err } } - if err := pruneLogKeyCollector.Collect(k, nil); err != nil { - return err - } + + // if err := pruneLogKeyCollector.Collect(k, nil); err != nil { + // return err + // } + log.Info("[SPIDERMAN] Deleting kv.Log", "key", k) + c.DeleteCurrent() } } } @@ -526,8 +534,8 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune if err := pruneOldLogChunks(tx, kv.LogAddressIndex, addrs, pruneTo, ctx); err != nil { return err } - if err := pruneOldLogChunks(tx, kv.Log, pruneLogKeyCollector, pruneTo, ctx); err != nil { - return err - } + // if err := pruneOldLogChunks(tx, kv.Log, pruneLogKeyCollector, pruneTo, ctx); err != nil { + // return err + // } return nil } From caeb6e99155958b484ec37436e02fceecdcfc910 Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Mon, 22 Apr 2024 13:40:04 +0400 Subject: [PATCH 2/7] Cleanup --- eth/stagedsync/stage_log_index.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index 2c6b5ac4584..60741282586 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -395,11 +395,8 @@ func pruneOldLogChunks(tx kv.RwTx, bucket string, inMem *etl.Collector, pruneTo return err } var blockNum uint64 - // if bucket == kv.Log { - // blockNum = binary.BigEndian.Uint64(k) - // } else { - blockNum = uint64(binary.BigEndian.Uint32(k[len(key):])) - // } + blockNum = uint64(binary.BigEndian.Uint32(k[len(key):])) + if !bytes.HasPrefix(k, key) || blockNum >= pruneTo { break } @@ -460,8 +457,6 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune defer topics.Close() addrs := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger) defer addrs.Close() - // pruneLogKeyCollector := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger) - // defer pruneLogKeyCollector.Close() reader := bytes.NewReader(nil) { @@ -505,7 +500,6 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune break } } - log.Info("[SPIDERMAN] Pruneloop", "toPrune", toPrune) if toPrune { for _, l := range logs { @@ -518,10 +512,6 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune return err } } - - // if err := pruneLogKeyCollector.Collect(k, nil); err != nil { - // return err - // } log.Info("[SPIDERMAN] Deleting kv.Log", "key", k) c.DeleteCurrent() } @@ -534,8 +524,5 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune if err := pruneOldLogChunks(tx, kv.LogAddressIndex, addrs, pruneTo, ctx); err != nil { return err } - // if err := pruneOldLogChunks(tx, kv.Log, pruneLogKeyCollector, pruneTo, ctx); err != nil { - // return err - // } return nil } From fb4bda8806c1ae92f124756e5ff8c1264c35b43c Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Mon, 22 Apr 2024 13:41:55 +0400 Subject: [PATCH 3/7] Cleanup --- eth/stagedsync/stage_log_index.go | 1 - 1 file changed, 1 deletion(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index 60741282586..44155e5ee33 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -512,7 +512,6 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune return err } } - log.Info("[SPIDERMAN] Deleting kv.Log", "key", k) c.DeleteCurrent() } } From d734b52035b2e80ab56bb920eaa9e26be3041168 Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Mon, 22 Apr 2024 13:49:08 +0400 Subject: [PATCH 4/7] Cleanup --- eth/stagedsync/stage_log_index.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index 44155e5ee33..3de6c99621f 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -431,7 +431,6 @@ func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte } pruneTo := cfg.prune.Receipts.PruneTo(s.ForwardProgress) - // s.PruneProgress if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, s.PruneProgress, pruneTo, ctx, logger, cfg.noPruneContracts); err != nil { return err } @@ -466,20 +465,17 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune } defer c.Close() - log.Info("pruneLogIndex running", "pruneFrom", pruneFrom, "pruneTo", pruneTo) - for k, v, err := c.Seek(dbutils.LogKey(pruneFrom, 0)); k != nil; k, v, err = c.Next() { if err != nil { return err } blockNum := binary.BigEndian.Uint64(k) - log.Info("Pruneloop", "key", k, "blockNum", blockNum) if blockNum >= pruneTo { break } select { case <-logEvery.C: - logger.Info(fmt.Sprintf("[%s]", logPrefix), "table", kv.Log, "block", blockNum) + logger.Info(fmt.Sprintf("[%s]", logPrefix), "table", kv.Log, "block", blockNum, "pruneFrom", pruneFrom, "pruneTo", pruneTo) case <-ctx.Done(): return libcommon.ErrStopped default: From 611d927ae4ff620cbd547c19a18389c794e1cfbf Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Tue, 23 Apr 2024 11:04:37 +0400 Subject: [PATCH 5/7] Add err check --- eth/stagedsync/stage_log_index.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index 3de6c99621f..b4f463ff788 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -508,7 +508,9 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune return err } } - c.DeleteCurrent() + if err := c.DeleteCurrent(); err != nil { + return err + } } } } From f872c07a7a7236ba16c8f90c462cc1ebc22590d8 Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Tue, 23 Apr 2024 11:43:36 +0400 Subject: [PATCH 6/7] Switch to tx.Delete --- eth/stagedsync/stage_log_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index b4f463ff788..4719dc7e3fa 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -508,7 +508,7 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune return err } } - if err := c.DeleteCurrent(); err != nil { + if err := tx.Delete(kv.Log, k); err != nil { return err } } From 790ebf32eca8238c3fb219b7d09a616914af7b5e Mon Sep 17 00:00:00 2001 From: Somnath Banerjee Date: Tue, 23 Apr 2024 16:04:30 +0400 Subject: [PATCH 7/7] Switch back to read cursor --- eth/stagedsync/stage_log_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/stagedsync/stage_log_index.go b/eth/stagedsync/stage_log_index.go index 4719dc7e3fa..56e52353720 100644 --- a/eth/stagedsync/stage_log_index.go +++ b/eth/stagedsync/stage_log_index.go @@ -459,7 +459,7 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune reader := bytes.NewReader(nil) { - c, err := tx.RwCursor(kv.Log) + c, err := tx.Cursor(kv.Log) if err != nil { return err }