Skip to content

Commit

Permalink
erigon2: implemented chain switching by --chain flag (#3668)
Browse files Browse the repository at this point in the history 
erigon2: fix linter
  • Loading branch information
@awskii
awskii authored Mar 10, 2022
1 parent 04f07a0 commit d300c28
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 42 deletions.
121 changes: 81 additions & 40 deletions cmd/state/commands/erigon2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ import (
"github.com/ledgerwatch/erigon-lib/aggregator"
libcommon "github.com/ledgerwatch/erigon-lib/common"
kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx"
"github.com/ledgerwatch/log/v3"
"github.com/spf13/cobra"

"github.com/ledgerwatch/erigon/cmd/rpcdaemon/interfaces"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/consensus/ethash"
Expand All @@ -34,8 +37,6 @@ import (
"github.com/ledgerwatch/erigon/params"
"github.com/ledgerwatch/erigon/turbo/snapshotsync"
"github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
"github.com/ledgerwatch/log/v3"
"github.com/spf13/cobra"
)

const (
Expand All @@ -55,6 +56,8 @@ func init() {
withBlock(erigon2Cmd)
withDataDir(erigon2Cmd)
withSnapshotBlocks(erigon2Cmd)
withChain(erigon2Cmd)

erigon2Cmd.Flags().BoolVar(&changesets, "changesets", false, "set to true to generate changesets")
erigon2Cmd.Flags().IntVar(&commitmentFrequency, "commfreq", 625, "how many blocks to skip between calculating commitment")
erigon2Cmd.Flags().BoolVar(&commitments, "commitments", false, "set to true to calculate commitments")
Expand All @@ -70,15 +73,15 @@ var erigon2Cmd = &cobra.Command{
return fmt.Errorf("commitmentFrequency cannot be less than 1 or more than %d: %d", commitmentFrequency, aggregationStep)
}
logger := log.New()
return Erigon2(genesis, logger)
return Erigon2(genesis, chainConfig, logger)
},
}

const (
logInterval = 30 * time.Second
)

func Erigon2(genesis *core.Genesis, logger log.Logger) error {
func Erigon2(genesis *core.Genesis, chainConfig *params.ChainConfig, logger log.Logger) error {
sigs := make(chan os.Signal, 1)
interruptCh := make(chan bool, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
Expand All @@ -87,6 +90,7 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
<-sigs
interruptCh <- true
}()

historyDb, err := kv2.NewMDBX(logger).Path(path.Join(datadir, "chaindata")).Open()
if err != nil {
return fmt.Errorf("opening chaindata as read only: %v", err)
Expand Down Expand Up @@ -116,8 +120,8 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
return fmt.Errorf("create aggregator: %w", err3)
}
defer agg.Close()
chainConfig := genesis.Config
vmConfig := vm.Config{}

var vmConfig vm.Config

var blockReader interfaces.FullBlockReader
if snapshotBlocks {
Expand Down Expand Up @@ -164,49 +168,31 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
}
}
}
blockNum := uint64(0)
var txNum uint64 = 1
trace := false

logger.Info("Initialised chain configuration", "config", chainConfig)

var (
blockNum uint64 = 0
txNum uint64 = 1
trace bool
)

logEvery := time.NewTicker(logInterval)
defer logEvery.Stop()
prevBlock := blockNum
prevTime := time.Now()
var prevHits, prevMisses uint64
var m runtime.MemStats

statx := &stat{
prevBlock: blockNum,
prevTime: time.Now(),
}

for !interrupt {
select {
default:
case <-logEvery.C:
aStats := agg.Stats()
totalFiles := aStats.AccountsCount + aStats.CodeCount + aStats.StorageCount + aStats.CommitmentCount
totalDatSize := aStats.AccountsDatSize + aStats.CodeDatSize + aStats.StorageDatSize + aStats.CommitmentDatSize
totalIdxSize := aStats.AccountsIdxSize + aStats.CodeIdxSize + aStats.StorageIdxSize + aStats.CommitmentIdxSize
currentTime := time.Now()
interval := currentTime.Sub(prevTime)
speed := float64(blockNum-prevBlock) / (float64(interval) / float64(time.Second))
prevBlock = blockNum
prevTime = currentTime
hits := aStats.Hits - prevHits
misses := aStats.Misses - prevMisses
prevHits = aStats.Hits
prevMisses = aStats.Misses
var hitRatio float64
if hits+misses > 0 {
hitRatio = float64(hits) / float64(hits+misses)
}
runtime.ReadMemStats(&m)
log.Info("Progress", "block", blockNum, "blk/s", speed, "state files", totalFiles,
"accounts", libcommon.ByteCount(uint64(aStats.AccountsDatSize+aStats.AccountsIdxSize)),
"code", libcommon.ByteCount(uint64(aStats.CodeDatSize+aStats.CodeIdxSize)),
"storage", libcommon.ByteCount(uint64(aStats.StorageDatSize+aStats.StorageIdxSize)),
"commitment", libcommon.ByteCount(uint64(aStats.CommitmentDatSize+aStats.CommitmentIdxSize)),
"total dat", libcommon.ByteCount(uint64(totalDatSize)), "total idx", libcommon.ByteCount(uint64(totalIdxSize)),
"hit ratio", hitRatio, "hits+misses", hits+misses,
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys),
)
statx.delta(aStats, blockNum).print(aStats, logger)
}
blockNum++
//fmt.Printf("block %d\n", blockNum)
trace = traceBlock != 0 && blockNum == uint64(traceBlock)
blockHash, err := blockReader.CanonicalHash(ctx, historyTx, blockNum)
if err != nil {
Expand All @@ -224,6 +210,7 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
return err
}
if b == nil {
log.Info("history: block is nil", "block", blockNum)
break
}
r := agg.MakeStateReader(blockNum)
Expand Down Expand Up @@ -267,12 +254,66 @@ func Erigon2(genesis *core.Genesis, logger log.Logger) error {
return err
}
}

aStats := agg.Stats()
statx.delta(aStats, blockNum).print(aStats, logger)
if w != nil {
w.Close()
}
return nil
}

type stat struct {
blockNum uint64
hits uint64
misses uint64
prevBlock uint64
prevMisses uint64
prevHits uint64
hitMissRatio float64
speed float64
prevTime time.Time
mem runtime.MemStats
}

func (s *stat) print(aStats aggregator.FilesStats, logger log.Logger) {
totalFiles := aStats.AccountsCount + aStats.CodeCount + aStats.StorageCount + aStats.CommitmentCount
totalDatSize := aStats.AccountsDatSize + aStats.CodeDatSize + aStats.StorageDatSize + aStats.CommitmentDatSize
totalIdxSize := aStats.AccountsIdxSize + aStats.CodeIdxSize + aStats.StorageIdxSize + aStats.CommitmentIdxSize

logger.Info("Progress", "block", s.blockNum, "blk/s", s.speed, "state files", totalFiles,
"accounts", libcommon.ByteCount(uint64(aStats.AccountsDatSize+aStats.AccountsIdxSize)),
"code", libcommon.ByteCount(uint64(aStats.CodeDatSize+aStats.CodeIdxSize)),
"storage", libcommon.ByteCount(uint64(aStats.StorageDatSize+aStats.StorageIdxSize)),
"commitment", libcommon.ByteCount(uint64(aStats.CommitmentDatSize+aStats.CommitmentIdxSize)),
"total dat", libcommon.ByteCount(uint64(totalDatSize)), "total idx", libcommon.ByteCount(uint64(totalIdxSize)),
"hit ratio", s.hitMissRatio, "hits+misses", s.hits+s.misses,
"alloc", libcommon.ByteCount(s.mem.Alloc), "sys", libcommon.ByteCount(s.mem.Sys),
)
}

func (s *stat) delta(aStats aggregator.FilesStats, blockNum uint64) *stat {
currentTime := time.Now()
runtime.ReadMemStats(&s.mem)

interval := currentTime.Sub(s.prevTime).Seconds()
s.blockNum = blockNum
s.speed = float64(s.blockNum-s.prevBlock) / interval
s.prevBlock = blockNum
s.prevTime = currentTime

s.hits = aStats.Hits - s.prevHits
s.misses = aStats.Misses - s.prevMisses
s.prevHits = aStats.Hits
s.prevMisses = aStats.Misses

total := s.hits + s.misses
if total > 0 {
s.hitMissRatio = float64(s.hits) / float64(total)
}
return s
}

func runBlock2(trace bool, txNumStart uint64, rw *ReaderWrapper, ww *WriterWrapper, chainConfig *params.ChainConfig, getHeader func(hash common.Hash, number uint64) *types.Header, block *types.Block, vmConfig vm.Config) (uint64, types.Receipts, error) {
defer blockExecutionTimer.UpdateDuration(time.Now())
header := block.Header()
Expand Down
6 changes: 6 additions & 0 deletions cmd/state/commands/global_flags_vars.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package commands
import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/common/paths"

"github.com/spf13/cobra"
)

Expand All @@ -14,6 +15,7 @@ var (
changeSetBucket string
indexBucket string
snapshotBlocks bool
chain string
)

func must(err error) {
Expand Down Expand Up @@ -50,3 +52,7 @@ func withIndexBucket(cmd *cobra.Command) {
func withSnapshotBlocks(cmd *cobra.Command) {
cmd.Flags().BoolVar(&snapshotBlocks, "experimental.snapshot", false, "")
}

func withChain(cmd *cobra.Command) {
cmd.Flags().StringVar(&chain, "chain", "", "pick a chain to assume (mainnet, ropsten, etc.)")
}
25 changes: 23 additions & 2 deletions cmd/state/commands/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,18 @@ import (
"path/filepath"

"github.com/ledgerwatch/erigon-lib/common"
"github.com/spf13/cobra"

"github.com/ledgerwatch/erigon/cmd/utils"
"github.com/ledgerwatch/erigon/core"
"github.com/ledgerwatch/erigon/internal/debug"
"github.com/spf13/cobra"
"github.com/ledgerwatch/erigon/params"
)

var (
genesisPath string
genesis *core.Genesis
chainConfig *params.ChainConfig
)

func init() {
Expand All @@ -31,10 +34,19 @@ var rootCmd = &cobra.Command{
panic(err)
}

genesis = core.DefaultGenesisBlock()
genesis, chainConfig = getChainGenesisAndConfig()
if genesisPath != "" {
genesis = genesisFromFile(genesisPath)
}
if genesis.Config != nil && genesis.Config.ChainID.Cmp(chainConfig.ChainID) != 0 {
utils.Fatalf("provided genesis.json chain configuration is invalid: expected chainId to be %v, got %v",
chainConfig.ChainID.String(), genesis.Config.ChainID.String())
}
// Apply special hacks for BSC params
if chainConfig.Parlia != nil {
params.ApplyBinanceSmartChainParams()
}

if chaindata == "" {
chaindata = filepath.Join(datadir, "chaindata")
}
Expand All @@ -58,6 +70,15 @@ func genesisFromFile(genesisPath string) *core.Genesis {
return genesis
}

func getChainGenesisAndConfig() (genesis *core.Genesis, chainConfig *params.ChainConfig) {
if chain == "" {
genesis, chainConfig = core.DefaultGenesisBlock(), params.MainnetChainConfig
} else {
genesis, chainConfig = core.DefaultGenesisBlockByChainName(chain), params.ChainConfigByChainName(chain)
}
return genesis, chainConfig
}

func Execute() {
ctx, cancel := common.RootContext()
defer cancel()
Expand Down

0 comments on commit d300c28

Please sign in to comment.