From f11a49a120380cf0b80bb5c090b64e6763c97e1b Mon Sep 17 00:00:00 2001 From: zenground0 Date: Wed, 25 Dec 2024 21:18:11 +0530 Subject: [PATCH 01/20] splitstore keys info command --- cmd/lotus-shed/splitstore.go | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index e8c45a0c5e8..eaf218eefda 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "encoding/binary" "fmt" "io" "os" @@ -31,6 +32,7 @@ var splitstoreCmd = &cli.Command{ splitstoreClearCmd, splitstoreCheckCmd, splitstoreInfoCmd, + splitstoreRepoInfoCmd, }, } @@ -353,6 +355,40 @@ func deleteSplitstoreKeys(lr repo.LockedRepo) error { return nil } +func printSplitstoreKeys(lr repo.LockedRepo) error { + ds, err := lr.Datastore(context.TODO(), "/metadata") + if err != nil { + return xerrors.Errorf("error opening datastore: %w", err) + } + if closer, ok := ds.(io.Closer); ok { + defer closer.Close() //nolint + } + + res, err := ds.Query(context.Background(), query.Query{Prefix: "/splitstore"}) + if err != nil { + return xerrors.Errorf("error querying datastore for splitstore keys: %w", err) + } + + fmt.Println("Splitstore keys and values:") + for r := range res.Next() { + if r.Error != nil { + return xerrors.Errorf("datastore query error: %w", r.Error) + } + + // Get the value for this key + value, err := ds.Get(context.Background(), datastore.NewKey(r.Key)) + if err != nil { + return xerrors.Errorf("error getting value for key %s: %w", r.Key, err) + } + + // Decode the value as a uvarint + decoded, _ := binary.Uvarint(value) + fmt.Printf(" %s: %d\n", r.Key, decoded) + } + + return nil +} + // badger logging through go-log type badgerLogger struct { *zap.SugaredLogger @@ -378,6 +414,39 @@ var splitstoreCheckCmd = &cli.Command{ }, } +var splitstoreRepoInfoCmd = &cli.Command{ + Name: "splitstore-info", + Usage: "Display splitstore metadata information", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + Usage: "lotus repo path", + }, + }, + Action: func(cctx *cli.Context) error { + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("error opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return xerrors.Errorf("error locking repo: %w", err) + } + + return printSplitstoreKeys(lr) + }, +} + var splitstoreInfoCmd = &cli.Command{ Name: "info", Description: "prints some basic splitstore information", From f593bb1f3eedca4b850765eabe1576f41c670078 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 00:06:45 +0530 Subject: [PATCH 02/20] load snapshot to hotstore and skip warmup --- blockstore/splitstore/splitstore_warmup.go | 191 +++++++++++---------- cmd/lotus/daemon.go | 2 +- go.mod | 27 ++- go.sum | 48 +++++- node/repo/fsrepo.go | 75 +++++--- 5 files changed, 218 insertions(+), 125 deletions(-) diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index 5a8dbc4d6ce..c8b9df26f30 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -1,17 +1,11 @@ package splitstore import ( - "sync" "sync/atomic" "time" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" "golang.org/x/xerrors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" ) @@ -35,7 +29,7 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { log.Info("warming up hotstore") start := time.Now() - err := s.doWarmup(curTs) + err := s.doWarmup2(curTs) if err != nil { log.Errorf("error warming up hotstore: %s", err) return @@ -47,86 +41,14 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { return nil } -// the actual warmup procedure; it walks the chain loading all state roots at the boundary -// and headers all the way up to genesis. -// objects are written in batches so as to minimize overhead. -func (s *SplitStore) doWarmup(curTs *types.TipSet) error { - var boundaryEpoch abi.ChainEpoch +func (s *SplitStore) doWarmup2(curTs *types.TipSet) error { + log.Infow("warmup starting") + epoch := curTs.Height() - if WarmupBoundary < epoch { - boundaryEpoch = epoch - WarmupBoundary - } - var mx sync.Mutex - batchHot := make([]blocks.Block, 0, batchSize) count := new(int64) - xcount := new(int64) - missing := new(int64) - - visitor, err := s.markSetEnv.New("warmup", 0) - if err != nil { - return xerrors.Errorf("error creating visitor: %w", err) - } - defer visitor.Close() //nolint - - err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup - visitor, - func(c cid.Cid) error { - if isUnitaryObject(c) { - return errStopWalk - } - - atomic.AddInt64(count, 1) - - has, err := s.hot.Has(s.ctx, c) - if err != nil { - return err - } - - if has { - return nil - } - - blk, err := s.cold.Get(s.ctx, c) - if err != nil { - if ipld.IsNotFound(err) { - atomic.AddInt64(missing, 1) - return errStopWalk - } - return err - } - - atomic.AddInt64(xcount, 1) - - mx.Lock() - batchHot = append(batchHot, blk) - if len(batchHot) == batchSize { - err = s.hot.PutMany(s.ctx, batchHot) - if err != nil { - mx.Unlock() - return err - } - batchHot = batchHot[:0] - } - mx.Unlock() - - return nil - }, func(cid.Cid) error { return nil }) - - if err != nil { - return err - } - - if len(batchHot) > 0 { - err = s.hot.PutMany(s.ctx, batchHot) - if err != nil { - return err - } - } - - log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing) - + *count = 10_000_000_000 s.markSetSize = *count + *count>>2 // overestimate a bit - err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) + err := s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) if err != nil { log.Warnf("error saving mark set size: %s", err) } @@ -136,7 +58,6 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error { if err != nil { return xerrors.Errorf("error saving warm up epoch: %w", err) } - s.warmupEpoch.Store(int64(epoch)) // also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes @@ -144,6 +65,104 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error { if err != nil { return xerrors.Errorf("error saving compaction index: %w", err) } - return nil } + +// the legacy warmup procedure before we wrote snapshots directly to the hotstore +// func (s *SplitStore) doWarmup(curTs *types.TipSet) error { +// var boundaryEpoch abi.ChainEpoch +// epoch := curTs.Height() +// if WarmupBoundary < epoch { +// boundaryEpoch = epoch - WarmupBoundary +// } +// var mx sync.Mutex +// batchHot := make([]blocks.Block, 0, batchSize) +// count := new(int64) +// xcount := new(int64) +// missing := new(int64) + +// visitor, err := s.markSetEnv.New("warmup", 0) +// if err != nil { +// return xerrors.Errorf("error creating visitor: %w", err) +// } +// defer visitor.Close() //nolint + +// err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup +// visitor, +// func(c cid.Cid) error { +// if isUnitaryObject(c) { +// return errStopWalk +// } + +// atomic.AddInt64(count, 1) + +// has, err := s.hot.Has(s.ctx, c) +// if err != nil { +// return err +// } + +// if has { +// return nil +// } + +// blk, err := s.cold.Get(s.ctx, c) +// if err != nil { +// if ipld.IsNotFound(err) { +// atomic.AddInt64(missing, 1) +// return errStopWalk +// } +// return err +// } + +// atomic.AddInt64(xcount, 1) + +// mx.Lock() +// batchHot = append(batchHot, blk) +// if len(batchHot) == batchSize { +// err = s.hot.PutMany(s.ctx, batchHot) +// if err != nil { +// mx.Unlock() +// return err +// } +// batchHot = batchHot[:0] +// } +// mx.Unlock() + +// return nil +// }, func(cid.Cid) error { return nil }) + +// if err != nil { +// return err +// } + +// if len(batchHot) > 0 { +// err = s.hot.PutMany(s.ctx, batchHot) +// if err != nil { +// return err +// } +// } + +// log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing) + +// s.markSetSize = *count + *count>>2 // overestimate a bit +// err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) +// if err != nil { +// log.Warnf("error saving mark set size: %s", err) +// } + +// // save the warmup epoch +// err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch)) +// if err != nil { +// return xerrors.Errorf("error saving warm up epoch: %w", err) +// } + +// s.warmupEpoch.Store(int64(epoch)) + +// // also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes +// err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex)) +// if err != nil { +// return xerrors.Errorf("error saving compaction index: %w", err) +// } + +// return nil +// } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 97d2294b79b..3d050af9fb0 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -546,7 +546,7 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) } defer lr.Close() //nolint:errcheck - bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, err := lr.Blockstore(ctx, repo.HotBlockstore) if err != nil { return xerrors.Errorf("failed to open blockstore: %w", err) } diff --git a/go.mod b/go.mod index f872e984121..ac7788be876 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/elastic/go-sysinfo v1.7.0 github.com/elastic/gosigar v0.14.3 github.com/etclabscore/go-openrpc-reflect v0.0.36 - github.com/fatih/color v1.15.0 + github.com/fatih/color v1.16.0 github.com/filecoin-project/filecoin-ffi v1.31.0 github.com/filecoin-project/go-address v1.2.0 github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 @@ -83,7 +83,7 @@ require ( github.com/hashicorp/golang-lru/arc/v2 v2.0.7 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 - github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab + github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c github.com/invopop/jsonschema v0.12.0 github.com/ipfs/bbloom v0.0.4 github.com/ipfs/boxo v0.20.0 @@ -136,7 +136,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // dependency-check-ignore: unknown github.com/triplewz/poseidon v0.0.2 - github.com/urfave/cli/v2 v2.25.5 + github.com/urfave/cli/v2 v2.25.7 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.2.0 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 @@ -211,7 +211,7 @@ require ( github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.3 // indirect github.com/go-openapi/jsonreference v0.19.4 // indirect github.com/go-openapi/swag v0.19.11 // indirect @@ -221,7 +221,7 @@ require ( github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect @@ -318,7 +318,7 @@ require ( github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect; dependency-check-ignore: unknown github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v2.18.12+incompatible // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartystreets/assertions v1.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -356,3 +356,18 @@ require ( lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +require github.com/ethereum/go-ethereum v1.14.12 + +require ( + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/deckarep/golang-set/v2 v2.6.0 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/holiman/uint256 v1.3.1 // indirect + github.com/supranational/blst v0.3.13 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect +) diff --git a/go.sum b/go.sum index 2b50a03dba9..92446fed1e6 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -141,6 +143,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -189,6 +193,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= @@ -199,6 +207,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= @@ -249,9 +259,15 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.36 h1:kSqNB2U8RVoW4si+4fsv13NGNkRAQ5j78zTUx1qiehk= github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= +github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -382,8 +398,9 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -463,8 +480,9 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -552,6 +570,8 @@ github.com/hashicorp/golang-lru/arc/v2 v2.0.7/go.mod h1:Pe7gBlGdc8clY5LJ0LpJXMt5 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -570,8 +590,8 @@ github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lTo github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab h1:HqW4xhhynfjrtEiiSGcQUd6vrK23iMam1FO8rI7mwig= -github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs= +github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= @@ -1030,6 +1050,8 @@ github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOW github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1188,8 +1210,9 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn github.com/sercand/kuberesolver/v4 v4.0.0 h1:frL7laPDG/lFm5n98ODmWnn+cvPpzlkf3LhzuPhcHP4= github.com/sercand/kuberesolver/v4 v4.0.0/go.mod h1:F4RGyuRmMAjeXHKL+w4P7AwUnPceEAPAhxUgXZjKgvM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -1270,6 +1293,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= +github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -1285,6 +1310,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/triplewz/poseidon v0.0.2 h1:s5QMVYnUfqvgM1eIqp7O9hHjZLVrKnkhx0E7EQTf9Nk= github.com/triplewz/poseidon v0.0.2/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -1294,8 +1323,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= -github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= @@ -1672,6 +1701,7 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 1c2e9f738d4..2737020b896 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -369,12 +369,15 @@ type fsLockedRepo struct { dsErr error dsOnce sync.Once - bs blockstore.Blockstore - bsErr error - bsOnce sync.Once - ssPath string - ssErr error - ssOnce sync.Once + bs blockstore.Blockstore + bsOnce sync.Once + bsErr error + bsHot blockstore.Blockstore + bsHotOnce sync.Once + bsHotErr error + ssPath string + ssErr error + ssOnce sync.Once chainIndexPath string chainIndexErr error @@ -424,38 +427,64 @@ func (fsr *fsLockedRepo) Close() error { // Blockstore returns a blockstore for the provided data domain. func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) { - if domain != UniversalBlockstore { + if domain != UniversalBlockstore && domain != HotBlockstore { return nil, ErrInvalidBlockstoreDomain } + if domain == UniversalBlockstore { + fsr.bsOnce.Do(func() { + path := fsr.join(filepath.Join(fsDatastore, "chain")) + readonly := fsr.readonly - fsr.bsOnce.Do(func() { - path := fsr.join(filepath.Join(fsDatastore, "chain")) - readonly := fsr.readonly + if err := os.MkdirAll(path, 0755); err != nil { + fsr.bsErr = err + return + } + opts, err := BadgerBlockstoreOptions(domain, path, readonly) + if err != nil { + fsr.bsErr = err + return + } + + if system.BadgerFsyncDisable { + opts.SyncWrites = false + } + + bs, err := badgerbs.Open(opts) + if err != nil { + fsr.bsErr = err + return + } + fsr.bs = blockstore.WrapIDStore(bs) + }) + return fsr.bs, fsr.bsErr + } + // else domain == HotBlockstore { + fsr.bsHotOnce.Do(func() { + path, err := fsr.SplitstorePath() + if err != nil { + fsr.bsHotErr = err + return + } + path = filepath.Join(path, "hot.badger") if err := os.MkdirAll(path, 0755); err != nil { - fsr.bsErr = err + fsr.bsHotErr = err return } - - opts, err := BadgerBlockstoreOptions(domain, path, readonly) + readonly := fsr.readonly + opts, err := BadgerBlockstoreOptions(HotBlockstore, path, readonly) if err != nil { - fsr.bsErr = err + fsr.bsHotErr = err return } - - if system.BadgerFsyncDisable { - opts.SyncWrites = false - } - bs, err := badgerbs.Open(opts) if err != nil { - fsr.bsErr = err + fsr.bsHotErr = err return } - fsr.bs = blockstore.WrapIDStore(bs) + fsr.bsHot = bs }) - - return fsr.bs, fsr.bsErr + return fsr.bsHot, fsr.bsHotErr } func (fsr *fsLockedRepo) SplitstorePath() (string, error) { From cc0acca2290fadd3195a7df814972dcca450cd78 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 00:50:34 +0530 Subject: [PATCH 03/20] Close hotstore after writing snapshot --- chain/gen/gen.go | 2 +- chain/store/store_test.go | 2 +- cmd/lotus-shed/balances.go | 4 +-- cmd/lotus-shed/deal-label.go | 2 +- cmd/lotus-shed/diff.go | 2 +- cmd/lotus-shed/export-car.go | 2 +- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/gas-estimation.go | 4 +-- cmd/lotus-shed/import-car.go | 4 +-- cmd/lotus-shed/invariants.go | 2 +- cmd/lotus-shed/miner-peerid.go | 2 +- cmd/lotus-shed/miner-types.go | 2 +- cmd/lotus-shed/msig.go | 2 +- cmd/lotus-shed/pruning.go | 2 +- cmd/lotus-shed/state-stats.go | 2 +- cmd/lotus-shed/terminations.go | 2 +- cmd/lotus/daemon.go | 3 +- node/modules/blockstore.go | 13 +++++---- node/repo/fsrepo.go | 50 ++++++++++++++------------------ node/repo/interface.go | 2 +- node/repo/memrepo.go | 6 ++-- 21 files changed, 54 insertions(+), 58 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 6747ff4098a..4ee83b7e4fc 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -134,7 +134,7 @@ func NewGeneratorWithSectorsAndUpgradeSchedule(numSectors int, us stmgr.UpgradeS return nil, xerrors.Errorf("failed to get metadata datastore: %w", err) } - bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore) if err != nil { return nil, err } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index 5329371ae33..5925ca4a906 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -56,7 +56,7 @@ func BenchmarkGetRandomness(b *testing.B) { b.Fatal(err) } - bs, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(context.TODO(), repo.UniversalBlockstore) if err != nil { b.Fatal(err) } diff --git a/cmd/lotus-shed/balances.go b/cmd/lotus-shed/balances.go index 48030c58ccd..684b940d1a7 100644 --- a/cmd/lotus-shed/balances.go +++ b/cmd/lotus-shed/balances.go @@ -497,7 +497,7 @@ var chainBalanceStateCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } @@ -721,7 +721,7 @@ var chainPledgeCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return xerrors.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/deal-label.go b/cmd/lotus-shed/deal-label.go index 417d1370193..36bb9874f14 100644 --- a/cmd/lotus-shed/deal-label.go +++ b/cmd/lotus-shed/deal-label.go @@ -52,7 +52,7 @@ var dealLabelCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/diff.go b/cmd/lotus-shed/diff.go index bdd2126b6d0..d7237ac522a 100644 --- a/cmd/lotus-shed/diff.go +++ b/cmd/lotus-shed/diff.go @@ -81,7 +81,7 @@ var diffMinerStates = &cli.Command{ _ = lkrepo.Close() }(lkrepo) - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/export-car.go b/cmd/lotus-shed/export-car.go index 7659d981376..81b6bc11e6d 100644 --- a/cmd/lotus-shed/export-car.go +++ b/cmd/lotus-shed/export-car.go @@ -70,7 +70,7 @@ var exportCarCmd = &cli.Command{ lr, err := r.Lock(repo.FullNode) if err == nil { - bs, err = lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err = lr.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index e6d0c4e056f..f4b42358bbb 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -95,7 +95,7 @@ var exportChainCmd = &cli.Command{ defer fi.Close() //nolint:errcheck - bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/gas-estimation.go b/cmd/lotus-shed/gas-estimation.go index b1c61b62f2c..a26bf60ca1f 100644 --- a/cmd/lotus-shed/gas-estimation.go +++ b/cmd/lotus-shed/gas-estimation.go @@ -80,7 +80,7 @@ var gasTraceCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } @@ -177,7 +177,7 @@ var replayOfflineCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/import-car.go b/cmd/lotus-shed/import-car.go index 973e7b31b84..b621177157e 100644 --- a/cmd/lotus-shed/import-car.go +++ b/cmd/lotus-shed/import-car.go @@ -47,7 +47,7 @@ var importCarCmd = &cli.Command{ return xerrors.Errorf("opening the car file: %w", err) } - bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return err } @@ -118,7 +118,7 @@ var importObjectCmd = &cli.Command{ } defer lr.Close() //nolint:errcheck - bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/invariants.go b/cmd/lotus-shed/invariants.go index bc994bbe633..9ba6292c3e0 100644 --- a/cmd/lotus-shed/invariants.go +++ b/cmd/lotus-shed/invariants.go @@ -79,7 +79,7 @@ var invariantsCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + cold, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open universal blockstore %w", err) } diff --git a/cmd/lotus-shed/miner-peerid.go b/cmd/lotus-shed/miner-peerid.go index e430637976c..3d620fdab2b 100644 --- a/cmd/lotus-shed/miner-peerid.go +++ b/cmd/lotus-shed/miner-peerid.go @@ -61,7 +61,7 @@ var minerPeeridCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/miner-types.go b/cmd/lotus-shed/miner-types.go index 822d037aa26..b53d5d58965 100644 --- a/cmd/lotus-shed/miner-types.go +++ b/cmd/lotus-shed/miner-types.go @@ -58,7 +58,7 @@ var minerTypesCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/msig.go b/cmd/lotus-shed/msig.go index ccc932c93ff..918d02f0d41 100644 --- a/cmd/lotus-shed/msig.go +++ b/cmd/lotus-shed/msig.go @@ -70,7 +70,7 @@ var multisigGetAllCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index c0bd453b145..b67b875d810 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -131,7 +131,7 @@ var stateTreePruneCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus-shed/state-stats.go b/cmd/lotus-shed/state-stats.go index 88b21f40767..399bae570ed 100644 --- a/cmd/lotus-shed/state-stats.go +++ b/cmd/lotus-shed/state-stats.go @@ -234,7 +234,7 @@ func loadChainStore(ctx context.Context, repoPath string) (*StoreHandle, error) return nil, err } - cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + cold, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return nil, xerrors.Errorf("failed to open universal blockstore %w", err) } diff --git a/cmd/lotus-shed/terminations.go b/cmd/lotus-shed/terminations.go index 563c1ba3a77..e67d3831042 100644 --- a/cmd/lotus-shed/terminations.go +++ b/cmd/lotus-shed/terminations.go @@ -61,7 +61,7 @@ var terminationsCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open blockstore: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 3d050af9fb0..d9901b29b00 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -546,10 +546,11 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) } defer lr.Close() //nolint:errcheck - bs, err := lr.Blockstore(ctx, repo.HotBlockstore) + bs, closer, err := lr.Blockstore(ctx, repo.HotBlockstore) if err != nil { return xerrors.Errorf("failed to open blockstore: %w", err) } + defer closer() mds, err := lr.Datastore(ctx, "/metadata") if err != nil { diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 9c54d51e60f..ec0d6fa5c03 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -23,7 +23,7 @@ import ( // chain data and state data. It can be backed by a blockstore directly // (e.g. Badger), or by a Splitstore. func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.UniversalBlockstore, error) { - bs, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore) + bs, _, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore) if err != nil { return nil, err } @@ -74,11 +74,12 @@ func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlocksto return bs, nil } -func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) { - return func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) { +func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, func() error, error) { + return func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, func() error, error) { + noop := func() error { return nil } path, err := r.SplitstorePath() if err != nil { - return nil, err + return nil, noop, err } cfg := &splitstore.Config{ @@ -93,7 +94,7 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked } ss, err := splitstore.Open(path, ds, hot, cold, cfg) if err != nil { - return nil, err + return nil, noop, err } lc.Append(fx.Hook{ OnStop: func(context.Context) error { @@ -101,7 +102,7 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked }, }) - return ss, err + return ss, noop, nil } } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 2737020b896..b769108d72c 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -426,9 +426,9 @@ func (fsr *fsLockedRepo) Close() error { } // Blockstore returns a blockstore for the provided data domain. -func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) { +func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, func() error, error) { if domain != UniversalBlockstore && domain != HotBlockstore { - return nil, ErrInvalidBlockstoreDomain + return nil, nil, ErrInvalidBlockstoreDomain } if domain == UniversalBlockstore { fsr.bsOnce.Do(func() { @@ -457,34 +457,28 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain } fsr.bs = blockstore.WrapIDStore(bs) }) - return fsr.bs, fsr.bsErr + return fsr.bs, nil, fsr.bsErr } // else domain == HotBlockstore { - fsr.bsHotOnce.Do(func() { - path, err := fsr.SplitstorePath() - if err != nil { - fsr.bsHotErr = err - return - } - path = filepath.Join(path, "hot.badger") - if err := os.MkdirAll(path, 0755); err != nil { - fsr.bsHotErr = err - return - } - readonly := fsr.readonly - opts, err := BadgerBlockstoreOptions(HotBlockstore, path, readonly) - if err != nil { - fsr.bsHotErr = err - return - } - bs, err := badgerbs.Open(opts) - if err != nil { - fsr.bsHotErr = err - return - } - fsr.bsHot = bs - }) - return fsr.bsHot, fsr.bsHotErr + path, err := fsr.SplitstorePath() + if err != nil { + return nil, nil, err + } + path = filepath.Join(path, "hot.badger") + if err := os.MkdirAll(path, 0755); err != nil { + return nil, nil, err + } + readonly := fsr.readonly + opts, err := BadgerBlockstoreOptions(HotBlockstore, path, readonly) + if err != nil { + return nil, nil, err + } + bs, err := badgerbs.Open(opts) + if err != nil { + return nil, nil, err + } + + return bs, bs.Close, nil } func (fsr *fsLockedRepo) SplitstorePath() (string, error) { diff --git a/node/repo/interface.go b/node/repo/interface.go index 100d0dc58d5..056b104c6c0 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -64,7 +64,7 @@ type LockedRepo interface { // The supplied context must only be used to initialize the blockstore. // The implementation should not retain the context for usage throughout // the lifecycle. - Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) + Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, func() error, error) // SplitstorePath returns the path for the SplitStore SplitstorePath() (string, error) diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index cda00f985f2..d602ae8e45f 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -253,11 +253,11 @@ func (lmem *lockedMemRepo) Datastore(_ context.Context, ns string) (datastore.Ba return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, error) { +func (lmem *lockedMemRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, func() error, error) { if domain != UniversalBlockstore { - return nil, ErrInvalidBlockstoreDomain + return nil, nil, ErrInvalidBlockstoreDomain } - return lmem.mem.blockstore, nil + return lmem.mem.blockstore, func() error { return nil }, nil } func (lmem *lockedMemRepo) SplitstorePath() (string, error) { From a4ac6106160c1e9b37445e7abc7123c9da83f431 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 10:42:03 +0530 Subject: [PATCH 04/20] Allow configuration of old style warmup routine --- blockstore/splitstore/splitstore.go | 4 + blockstore/splitstore/splitstore_warmup.go | 217 +++++++++++---------- cmd/lotus-sim/simulation/node.go | 2 +- node/config/def.go | 1 + node/config/types.go | 6 + node/modules/blockstore.go | 8 +- 6 files changed, 134 insertions(+), 104 deletions(-) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index a3b93599f00..18aa755bd21 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -102,6 +102,10 @@ type Config struct { // from the hotstore should be written to the cold store UniversalColdBlocks bool + // FullWarmup indicates to do a chain traversal upon splitstore init to copy + // from cold store to hot store + FullWarmup bool + // HotstoreMessageRetention indicates the hotstore retention policy for messages. // It has the following semantics: // - a value of 0 will only retain messages within the compaction boundary (4 finalities) diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index c8b9df26f30..4319184ace4 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -1,18 +1,25 @@ package splitstore import ( + "sync" "sync/atomic" "time" "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" ) var ( // WarmupBoundary is the number of epochs to load state during warmup. WarmupBoundary = policy.ChainFinality + // Empirically taken from December 2024 + MarkSetEstimate int64 = 10_000_000_000 ) // warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore; @@ -29,7 +36,12 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { log.Info("warming up hotstore") start := time.Now() - err := s.doWarmup2(curTs) + var err error + if s.cfg.FullWarmup { + err = s.doWarmup(curTs) + } else { + err = s.doWarmup2(curTs) + } if err != nil { log.Errorf("error warming up hotstore: %s", err) return @@ -41,12 +53,14 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { return nil } +// Warmup2 func (s *SplitStore) doWarmup2(curTs *types.TipSet) error { log.Infow("warmup starting") epoch := curTs.Height() count := new(int64) - *count = 10_000_000_000 + // Empirically taken from December 2024 + *count = MarkSetEstimate s.markSetSize = *count + *count>>2 // overestimate a bit err := s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) if err != nil { @@ -68,101 +82,104 @@ func (s *SplitStore) doWarmup2(curTs *types.TipSet) error { return nil } -// the legacy warmup procedure before we wrote snapshots directly to the hotstore -// func (s *SplitStore) doWarmup(curTs *types.TipSet) error { -// var boundaryEpoch abi.ChainEpoch -// epoch := curTs.Height() -// if WarmupBoundary < epoch { -// boundaryEpoch = epoch - WarmupBoundary -// } -// var mx sync.Mutex -// batchHot := make([]blocks.Block, 0, batchSize) -// count := new(int64) -// xcount := new(int64) -// missing := new(int64) - -// visitor, err := s.markSetEnv.New("warmup", 0) -// if err != nil { -// return xerrors.Errorf("error creating visitor: %w", err) -// } -// defer visitor.Close() //nolint - -// err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup -// visitor, -// func(c cid.Cid) error { -// if isUnitaryObject(c) { -// return errStopWalk -// } - -// atomic.AddInt64(count, 1) - -// has, err := s.hot.Has(s.ctx, c) -// if err != nil { -// return err -// } - -// if has { -// return nil -// } - -// blk, err := s.cold.Get(s.ctx, c) -// if err != nil { -// if ipld.IsNotFound(err) { -// atomic.AddInt64(missing, 1) -// return errStopWalk -// } -// return err -// } - -// atomic.AddInt64(xcount, 1) - -// mx.Lock() -// batchHot = append(batchHot, blk) -// if len(batchHot) == batchSize { -// err = s.hot.PutMany(s.ctx, batchHot) -// if err != nil { -// mx.Unlock() -// return err -// } -// batchHot = batchHot[:0] -// } -// mx.Unlock() - -// return nil -// }, func(cid.Cid) error { return nil }) - -// if err != nil { -// return err -// } - -// if len(batchHot) > 0 { -// err = s.hot.PutMany(s.ctx, batchHot) -// if err != nil { -// return err -// } -// } - -// log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing) - -// s.markSetSize = *count + *count>>2 // overestimate a bit -// err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) -// if err != nil { -// log.Warnf("error saving mark set size: %s", err) -// } - -// // save the warmup epoch -// err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch)) -// if err != nil { -// return xerrors.Errorf("error saving warm up epoch: %w", err) -// } - -// s.warmupEpoch.Store(int64(epoch)) - -// // also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes -// err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex)) -// if err != nil { -// return xerrors.Errorf("error saving compaction index: %w", err) -// } - -// return nil -// } +// the full warmup procedure +// this was standard warmup before we wrote snapshots directly to the hotstore +// now this is used only if explicitly configured. A good use case for this is +// when starting splitstore off of an unpruned full node. +func (s *SplitStore) doWarmup(curTs *types.TipSet) error { + var boundaryEpoch abi.ChainEpoch + epoch := curTs.Height() + if WarmupBoundary < epoch { + boundaryEpoch = epoch - WarmupBoundary + } + var mx sync.Mutex + batchHot := make([]blocks.Block, 0, batchSize) + count := new(int64) + xcount := new(int64) + missing := new(int64) + + visitor, err := s.markSetEnv.New("warmup", 0) + if err != nil { + return xerrors.Errorf("error creating visitor: %w", err) + } + defer visitor.Close() //nolint + + err = s.walkChain(curTs, boundaryEpoch, epoch+1, // we don't load messages/receipts in warmup + visitor, + func(c cid.Cid) error { + if isUnitaryObject(c) { + return errStopWalk + } + + atomic.AddInt64(count, 1) + + has, err := s.hot.Has(s.ctx, c) + if err != nil { + return err + } + + if has { + return nil + } + + blk, err := s.cold.Get(s.ctx, c) + if err != nil { + if ipld.IsNotFound(err) { + atomic.AddInt64(missing, 1) + return errStopWalk + } + return err + } + + atomic.AddInt64(xcount, 1) + + mx.Lock() + batchHot = append(batchHot, blk) + if len(batchHot) == batchSize { + err = s.hot.PutMany(s.ctx, batchHot) + if err != nil { + mx.Unlock() + return err + } + batchHot = batchHot[:0] + } + mx.Unlock() + + return nil + }, func(cid.Cid) error { return nil }) + + if err != nil { + return err + } + + if len(batchHot) > 0 { + err = s.hot.PutMany(s.ctx, batchHot) + if err != nil { + return err + } + } + + log.Infow("warmup stats", "visited", *count, "warm", *xcount, "missing", *missing) + + s.markSetSize = *count + *count>>2 // overestimate a bit + err = s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) + if err != nil { + log.Warnf("error saving mark set size: %s", err) + } + + // save the warmup epoch + err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch)) + if err != nil { + return xerrors.Errorf("error saving warm up epoch: %w", err) + } + + s.warmupEpoch.Store(int64(epoch)) + + // also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes + err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex)) + if err != nil { + return xerrors.Errorf("error saving compaction index: %w", err) + } + + return nil +} diff --git a/cmd/lotus-sim/simulation/node.go b/cmd/lotus-sim/simulation/node.go index cda3e69d839..f1efb2543cd 100644 --- a/cmd/lotus-sim/simulation/node.go +++ b/cmd/lotus-sim/simulation/node.go @@ -51,7 +51,7 @@ func NewNode(ctx context.Context, r repo.Repo) (nd *Node, _err error) { } }() - bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore) + bs, _, err := lr.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return nil, err } diff --git a/node/config/def.go b/node/config/def.go index e6bdc04bdb8..3f0c30272b3 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -75,6 +75,7 @@ func DefaultFullNode() *FullNode { ColdStoreType: "discard", HotStoreType: "badger", MarkSetType: "badger", + FullWarmup: false, HotStoreFullGCFrequency: 20, HotStoreMaxSpaceTarget: 650_000_000_000, diff --git a/node/config/types.go b/node/config/types.go index 3824427b2af..626f74029f2 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -523,6 +523,12 @@ type Splitstore struct { // is set. Moving GC will not occur when total moving size exceeds // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer HotstoreMaxSpaceSafetyBuffer uint64 + + // Perform a full warmup from the coldstore to the hotstore upon splitstore startup. + // This is useful in the case you are migrating from a non-splitstore setup to splitstore. + // This should be false in the common case where a node is initialized from a snapshot + // since snapshots are loaded directly to the hotstore. + FullWarmup bool } // Full Node diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index ec0d6fa5c03..4fbc4dfa9b7 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -83,9 +83,11 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked } cfg := &splitstore.Config{ - MarkSetType: cfg.Splitstore.MarkSetType, - DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", - UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", + MarkSetType: cfg.Splitstore.MarkSetType, + DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", + UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", + FullWarmup: cfg.Splitstore.FullWarmup, + HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, HotstoreMaxSpaceTarget: cfg.Splitstore.HotStoreMaxSpaceTarget, From 4c002ef3b8150bd0a42bfcae6a58174b2e57525d Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 11:05:53 +0530 Subject: [PATCH 05/20] Cleanup hot blockstore domain opening --- node/modules/blockstore.go | 45 ++++++++-------------------- node/repo/fsrepo.go | 60 ++++++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 4fbc4dfa9b7..2ff18f956e5 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -2,16 +2,12 @@ package modules import ( "context" - "io" - "os" - "path/filepath" bstore "github.com/ipfs/boxo/blockstore" "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/blockstore" - badgerbs "github.com/filecoin-project/lotus/blockstore/badger" "github.com/filecoin-project/lotus/blockstore/splitstore" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -23,17 +19,16 @@ import ( // chain data and state data. It can be backed by a blockstore directly // (e.g. Badger), or by a Splitstore. func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.UniversalBlockstore, error) { - bs, _, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore) + bs, closer, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore) if err != nil { return nil, err } - if c, ok := bs.(io.Closer); ok { - lc.Append(fx.Hook{ - OnStop: func(_ context.Context) error { - return c.Close() - }, - }) - } + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { + return closer() + }, + }) + return bs, err } @@ -45,32 +40,16 @@ func DiscardColdBlockstore(lc fx.Lifecycle, bs dtypes.UniversalBlockstore) (dtyp return blockstore.NewDiscardStore(bs), nil } -func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) { - path, err := r.SplitstorePath() - if err != nil { - return nil, err - } - - path = filepath.Join(path, "hot.badger") - if err := os.MkdirAll(path, 0755); err != nil { - return nil, err - } - - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly()) - if err != nil { - return nil, err - } - - bs, err := badgerbs.Open(opts) +func BadgerHotBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.HotBlockstore, error) { + bs, closer, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.HotBlockstore) if err != nil { return nil, err } - lc.Append(fx.Hook{ OnStop: func(_ context.Context) error { - return bs.Close() - }}) - + return closer() + }, + }) return bs, nil } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index b769108d72c..07fbb95bef5 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -419,6 +419,11 @@ func (fsr *fsLockedRepo) Close() error { return xerrors.Errorf("could not close blockstore: %w", err) } } + if c, ok := fsr.bsHot.(io.Closer); ok && c != nil { + if err := c.Close(); err != nil { + return xerrors.Errorf("could not close blockstore: %w", err) + } + } err = fsr.closer.Close() fsr.closer = nil @@ -430,12 +435,14 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain if domain != UniversalBlockstore && domain != HotBlockstore { return nil, nil, ErrInvalidBlockstoreDomain } + var bs *badgerbs.Blockstore + var err error if domain == UniversalBlockstore { fsr.bsOnce.Do(func() { path := fsr.join(filepath.Join(fsDatastore, "chain")) readonly := fsr.readonly - if err := os.MkdirAll(path, 0755); err != nil { + if err = os.MkdirAll(path, 0755); err != nil { fsr.bsErr = err return } @@ -450,35 +457,44 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain opts.SyncWrites = false } - bs, err := badgerbs.Open(opts) + bs, err = badgerbs.Open(opts) if err != nil { fsr.bsErr = err return } fsr.bs = blockstore.WrapIDStore(bs) }) - return fsr.bs, nil, fsr.bsErr - } - // else domain == HotBlockstore { - path, err := fsr.SplitstorePath() - if err != nil { - return nil, nil, err - } - path = filepath.Join(path, "hot.badger") - if err := os.MkdirAll(path, 0755); err != nil { - return nil, nil, err - } - readonly := fsr.readonly - opts, err := BadgerBlockstoreOptions(HotBlockstore, path, readonly) - if err != nil { - return nil, nil, err - } - bs, err := badgerbs.Open(opts) - if err != nil { - return nil, nil, err + err = fsr.bsErr + } else { + // else domain == HotBlockstore { + fsr.bsHotOnce.Do(func() { + path, err := fsr.SplitstorePath() + if err != nil { + fsr.bsHotErr = err + return + } + path = filepath.Join(path, "hot.badger") + if err := os.MkdirAll(path, 0755); err != nil { + fsr.bsHotErr = err + return + } + readonly := fsr.readonly + opts, err := BadgerBlockstoreOptions(HotBlockstore, path, readonly) + if err != nil { + fsr.bsHotErr = err + return + } + bs, err = badgerbs.Open(opts) + if err != nil { + fsr.bsHotErr = err + return + } + fsr.bsHot = bs + }) + err = fsr.bsHotErr } - return bs, bs.Close, nil + return bs, bs.Close, err } func (fsr *fsLockedRepo) SplitstorePath() (string, error) { From de81964c63710f7a8589dc0b7d376fe50ed7ddbe Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 11:17:55 +0530 Subject: [PATCH 06/20] docsgen cli --- blockstore/splitstore/splitstore_warmup.go | 7 ++++--- documentation/en/default-lotus-config.toml | 9 +++++++++ node/config/doc_gen.go | 9 +++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index 4319184ace4..e57ea25a525 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -5,14 +5,15 @@ import ( "sync/atomic" "time" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - ipld "github.com/ipfs/go-ipld-format" ) var ( diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 41bc082da38..82ee6162284 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -219,6 +219,15 @@ # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACESAFETYBUFFER #HotstoreMaxSpaceSafetyBuffer = 50000000000 + # Perform a full warmup from the coldstore to the hotstore upon splitstore startup. + # This is useful in the case you are migrating from a non-splitstore setup to splitstore. + # This should be false in the common case where a node is initialized from a snapshot + # since snapshots are loaded directly to the hotstore. + # + # type: bool + # env var: LOTUS_CHAINSTORE_SPLITSTORE_FULLWARMUP + #FullWarmup = false + [Fevm] # EnableEthRPC enables eth_ RPC methods. diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index feab3d5b6e0..4f0068f6135 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1201,6 +1201,15 @@ exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold`, is set. Moving GC will not occur when total moving size exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, }, + { + Name: "FullWarmup", + Type: "bool", + + Comment: `Perform a full warmup from the coldstore to the hotstore upon splitstore startup. +This is useful in the case you are migrating from a non-splitstore setup to splitstore. +This should be false in the common case where a node is initialized from a snapshot +since snapshots are loaded directly to the hotstore.`, + }, }, "StorageMiner": { { From c508613b2b4ddf20684898393b41a34c48a8eac4 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 11:27:29 +0530 Subject: [PATCH 07/20] Fix: we should be able to sync snapshot to universal when splitstore disabled --- cmd/lotus-shed/migrations.go | 3 ++- cmd/lotus/daemon.go | 33 ++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index be0ef834c4d..56bb38a31bc 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -128,10 +128,11 @@ var migrationsCmd = &cli.Command{ defer lkrepo.Close() //nolint:errcheck - cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + cold, closer, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) if err != nil { return fmt.Errorf("failed to open universal blockstore %w", err) } + defer closer() path, err := lkrepo.SplitstorePath() if err != nil { diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index d9901b29b00..707cd49dd41 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/go-paramfetch" lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain/beacon/drand" @@ -544,13 +545,28 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) if err != nil { return err } - defer lr.Close() //nolint:errcheck - bs, closer, err := lr.Blockstore(ctx, repo.HotBlockstore) + c, err := lr.Config() if err != nil { - return xerrors.Errorf("failed to open blockstore: %w", err) + return err + } + cfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + + var bs blockstore.Blockstore + if cfg.Chainstore.EnableSplitstore { + bs, _, err = lr.Blockstore(ctx, repo.HotBlockstore) + if err != nil { + return xerrors.Errorf("failed to open hot blockstore: %w", err) + } + } else { + bs, _, err = lr.Blockstore(ctx, repo.UniversalBlockstore) + if err != nil { + return xerrors.Errorf("failed to open universalblockstore: %w", err) + } } - defer closer() mds, err := lr.Datastore(ctx, "/metadata") if err != nil { @@ -630,15 +646,6 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) return err } - c, err := lr.Config() - if err != nil { - return err - } - cfg, ok := c.(*config.FullNode) - if !ok { - return xerrors.Errorf("invalid config for repo, got: %T", c) - } - if !cfg.ChainIndexer.EnableIndexer { log.Info("chain indexer is disabled, not populating index from snapshot") return nil From 466750e3c6c687087d4c23c08d4b9761bfef0c4d Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 11:37:04 +0530 Subject: [PATCH 08/20] Debug log --- node/repo/fsrepo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 07fbb95bef5..f7f6652b099 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -433,6 +433,7 @@ func (fsr *fsLockedRepo) Close() error { // Blockstore returns a blockstore for the provided data domain. func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, func() error, error) { if domain != UniversalBlockstore && domain != HotBlockstore { + log.Error("invalid blockstore domain", domain) return nil, nil, ErrInvalidBlockstoreDomain } var bs *badgerbs.Blockstore From e83a27ae3fadd3bb7bca819d4dfe129dba9a328e Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 11:45:04 +0530 Subject: [PATCH 09/20] Testing repo must now accept hot blockstore domain --- node/repo/memrepo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index d602ae8e45f..5da2b1711bb 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -254,7 +254,7 @@ func (lmem *lockedMemRepo) Datastore(_ context.Context, ns string) (datastore.Ba } func (lmem *lockedMemRepo) Blockstore(ctx context.Context, domain BlockstoreDomain) (blockstore.Blockstore, func() error, error) { - if domain != UniversalBlockstore { + if domain != UniversalBlockstore && domain != HotBlockstore { return nil, nil, ErrInvalidBlockstoreDomain } return lmem.mem.blockstore, func() error { return nil }, nil From 271bb6c7fccbf8ea2ee8c6a783495e29aa13e894 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 13:15:10 +0530 Subject: [PATCH 10/20] hmm go mod troubles --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ac7788be876..166125cccc5 100644 --- a/go.mod +++ b/go.mod @@ -368,6 +368,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/supranational/blst v0.3.13 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index 92446fed1e6..487280391c0 100644 --- a/go.sum +++ b/go.sum @@ -1310,10 +1310,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/triplewz/poseidon v0.0.2 h1:s5QMVYnUfqvgM1eIqp7O9hHjZLVrKnkhx0E7EQTf9Nk= github.com/triplewz/poseidon v0.0.2/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= From a75f86202f6c7115e0b5e3382e5aeeef9313e24d Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 14:04:42 +0530 Subject: [PATCH 11/20] fix go mod trouble --- go.mod | 10 ---------- go.sum | 23 ----------------------- 2 files changed, 33 deletions(-) diff --git a/go.mod b/go.mod index 166125cccc5..71b9cef1a22 100644 --- a/go.mod +++ b/go.mod @@ -357,17 +357,7 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -require github.com/ethereum/go-ethereum v1.14.12 - require ( - github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect - github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect - github.com/deckarep/golang-set/v2 v2.6.0 // indirect - github.com/ethereum/c-kzg-4844 v1.0.0 // indirect - github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/holiman/uint256 v1.3.1 // indirect - github.com/supranational/blst v0.3.13 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect ) diff --git a/go.sum b/go.sum index 487280391c0..d2acf5d2e12 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= -github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= -github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -143,8 +141,6 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= -github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -193,10 +189,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= -github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= @@ -207,8 +199,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= -github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= @@ -259,12 +249,6 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.36 h1:kSqNB2U8RVoW4si+4fsv13NGNkRAQ5j78zTUx1qiehk= github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= -github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= -github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= -github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -570,8 +554,6 @@ github.com/hashicorp/golang-lru/arc/v2 v2.0.7/go.mod h1:Pe7gBlGdc8clY5LJ0LpJXMt5 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= -github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -1050,8 +1032,6 @@ github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOW github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1293,8 +1273,6 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= -github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= -github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -1701,7 +1679,6 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 0ec6a17c2dc961dd3af016eb2444bf5259406910 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 14:06:44 +0530 Subject: [PATCH 12/20] changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b096c616fb..dee7c87c353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ # UNRELEASED +- Sync snapshots directly to hotstore. Save 120 G of lotus disk space. ([filecoin-project/lotus#12800](https://github.com/filecoin-project/lotus/pull/12803)) - Add json output of tipsets to `louts chain list`. ([filecoin-project/lotus#12691](https://github.com/filecoin-project/lotus/pull/12691)) - Remove IPNI advertisement relay over pubsub via Lotus node as it now has been deprecated. ([filecoin-project/lotus#12768](https://github.com/filecoin-project/lotus/pull/12768) - During a network upgrade, log migration progress every 2 seconds so they are more helpful and informative. The `LOTUS_MIGRATE_PROGRESS_LOG_SECONDS` environment variable can be used to change this if needed. ([filecoin-project/lotus#12732](https://github.com/filecoin-project/lotus/pull/12732)) From 6c8cc58c48297fae8335495b693ef6060c608e19 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 14:24:51 +0530 Subject: [PATCH 13/20] fixup go mod --- go.mod | 9 ++------- go.sum | 10 ++-------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 71b9cef1a22..f461d318fd4 100644 --- a/go.mod +++ b/go.mod @@ -221,7 +221,7 @@ require ( github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect @@ -318,7 +318,7 @@ require ( github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect; dependency-check-ignore: unknown github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/shirou/gopsutil v2.18.12+incompatible // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/smartystreets/assertions v1.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -356,8 +356,3 @@ require ( lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -require ( - github.com/tklauser/go-sysconf v0.3.14 // indirect - github.com/tklauser/numcpus v0.8.0 // indirect -) diff --git a/go.sum b/go.sum index d2acf5d2e12..b6158f66458 100644 --- a/go.sum +++ b/go.sum @@ -464,9 +464,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= -github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -1190,9 +1189,8 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn github.com/sercand/kuberesolver/v4 v4.0.0 h1:frL7laPDG/lFm5n98ODmWnn+cvPpzlkf3LhzuPhcHP4= github.com/sercand/kuberesolver/v4 v4.0.0/go.mod h1:F4RGyuRmMAjeXHKL+w4P7AwUnPceEAPAhxUgXZjKgvM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -1288,10 +1286,6 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= -github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= -github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= -github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/triplewz/poseidon v0.0.2 h1:s5QMVYnUfqvgM1eIqp7O9hHjZLVrKnkhx0E7EQTf9Nk= github.com/triplewz/poseidon v0.0.2/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= From 8613cf87e86ce614084fd696f000bd7d61c9cb7b Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 15:44:42 +0530 Subject: [PATCH 14/20] Fixup linting --- cmd/lotus-shed/migrations.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 56bb38a31bc..c3cfa98a670 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -132,7 +132,11 @@ var migrationsCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open universal blockstore %w", err) } - defer closer() + defer func() { + if err := closer(); err != nil { + log.Warnf("failed to close universal blockstore: %s", err) + } + }() path, err := lkrepo.SplitstorePath() if err != nil { From bb6d93adad86ea51fdc50f34825efb45584be5df Mon Sep 17 00:00:00 2001 From: zenground0 Date: Thu, 26 Dec 2024 17:43:27 +0530 Subject: [PATCH 15/20] Fix hang with crazy map allocation --- blockstore/splitstore/markset_map.go | 5 +++++ blockstore/splitstore/splitstore_warmup.go | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/blockstore/splitstore/markset_map.go b/blockstore/splitstore/markset_map.go index 52efcb8a484..922800eafad 100644 --- a/blockstore/splitstore/markset_map.go +++ b/blockstore/splitstore/markset_map.go @@ -42,6 +42,11 @@ func NewMapMarkSetEnv(path string) (*MapMarkSetEnv, error) { func (e *MapMarkSetEnv) New(name string, sizeHint int64) (MarkSet, error) { path := filepath.Join(e.path, name) + // Limit map size to 1k if sizeHint exceeds this value + // This prevents accidental hanging when sizeHint is too large + if sizeHint > 1000 { + sizeHint = 1000 + } return &MapMarkSet{ set: make(map[string]struct{}, sizeHint), path: path, diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index e57ea25a525..14107510404 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -59,10 +59,7 @@ func (s *SplitStore) doWarmup2(curTs *types.TipSet) error { log.Infow("warmup starting") epoch := curTs.Height() - count := new(int64) - // Empirically taken from December 2024 - *count = MarkSetEstimate - s.markSetSize = *count + *count>>2 // overestimate a bit + s.markSetSize = MarkSetEstimate err := s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) if err != nil { log.Warnf("error saving mark set size: %s", err) From a8243219f479abc7cfb3298291e0a9a0edf9e421 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Fri, 27 Dec 2024 16:09:54 +0530 Subject: [PATCH 16/20] revert no-warmup --- blockstore/splitstore/splitstore.go | 4 -- blockstore/splitstore/splitstore_test.go | 4 +- blockstore/splitstore/splitstore_warmup.go | 43 +++------------------- node/config/def.go | 1 - node/config/types.go | 6 --- node/modules/blockstore.go | 1 - 6 files changed, 8 insertions(+), 51 deletions(-) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 18aa755bd21..a3b93599f00 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -102,10 +102,6 @@ type Config struct { // from the hotstore should be written to the cold store UniversalColdBlocks bool - // FullWarmup indicates to do a chain traversal upon splitstore init to copy - // from cold store to hot store - FullWarmup bool - // HotstoreMessageRetention indicates the hotstore retention policy for messages. // It has the following semantics: // - a value of 0 will only retain messages within the compaction boundary (4 finalities) diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index 23eadec1fbe..ad6639f77f2 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -163,11 +163,11 @@ func testSplitStore(t *testing.T, cfg *Config) { hotCnt := countBlocks(hot) if coldCnt != 2 { - t.Errorf("expected %d blocks, but got %d", 2, coldCnt) + t.Fatalf("expected %d blocks, but got %d", 2, coldCnt) } if hotCnt != 12 { - t.Errorf("expected %d blocks, but got %d", 12, hotCnt) + t.Fatalf("expected %d blocks, but got %d", 12, hotCnt) } // trigger a compaction diff --git a/blockstore/splitstore/splitstore_warmup.go b/blockstore/splitstore/splitstore_warmup.go index 14107510404..1dd64236d3c 100644 --- a/blockstore/splitstore/splitstore_warmup.go +++ b/blockstore/splitstore/splitstore_warmup.go @@ -19,8 +19,6 @@ import ( var ( // WarmupBoundary is the number of epochs to load state during warmup. WarmupBoundary = policy.ChainFinality - // Empirically taken from December 2024 - MarkSetEstimate int64 = 10_000_000_000 ) // warmup acquires the compaction lock and spawns a goroutine to warm up the hotstore; @@ -37,12 +35,8 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { log.Info("warming up hotstore") start := time.Now() - var err error - if s.cfg.FullWarmup { - err = s.doWarmup(curTs) - } else { - err = s.doWarmup2(curTs) - } + err := s.doWarmup(curTs) + if err != nil { log.Errorf("error warming up hotstore: %s", err) return @@ -54,36 +48,11 @@ func (s *SplitStore) warmup(curTs *types.TipSet) error { return nil } -// Warmup2 -func (s *SplitStore) doWarmup2(curTs *types.TipSet) error { - log.Infow("warmup starting") - - epoch := curTs.Height() - s.markSetSize = MarkSetEstimate - err := s.ds.Put(s.ctx, markSetSizeKey, int64ToBytes(s.markSetSize)) - if err != nil { - log.Warnf("error saving mark set size: %s", err) - } - - // save the warmup epoch - err = s.ds.Put(s.ctx, warmupEpochKey, epochToBytes(epoch)) - if err != nil { - return xerrors.Errorf("error saving warm up epoch: %w", err) - } - s.warmupEpoch.Store(int64(epoch)) - - // also save the compactionIndex, as this is used as an indicator of warmup for upgraded nodes - err = s.ds.Put(s.ctx, compactionIndexKey, int64ToBytes(s.compactionIndex)) - if err != nil { - return xerrors.Errorf("error saving compaction index: %w", err) - } - return nil -} - // the full warmup procedure -// this was standard warmup before we wrote snapshots directly to the hotstore -// now this is used only if explicitly configured. A good use case for this is -// when starting splitstore off of an unpruned full node. +// Most of this is unnecessary in the common case where we are starting from a snapshot +// since snapshots are now loaded directly to the hotstore. However this is safe and robust, +// handling all cases where we are transition from a universal setup to a splitstore setup. +// And warmup costs are only paid on init so it is not func (s *SplitStore) doWarmup(curTs *types.TipSet) error { var boundaryEpoch abi.ChainEpoch epoch := curTs.Height() diff --git a/node/config/def.go b/node/config/def.go index 3f0c30272b3..e6bdc04bdb8 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -75,7 +75,6 @@ func DefaultFullNode() *FullNode { ColdStoreType: "discard", HotStoreType: "badger", MarkSetType: "badger", - FullWarmup: false, HotStoreFullGCFrequency: 20, HotStoreMaxSpaceTarget: 650_000_000_000, diff --git a/node/config/types.go b/node/config/types.go index 626f74029f2..3824427b2af 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -523,12 +523,6 @@ type Splitstore struct { // is set. Moving GC will not occur when total moving size exceeds // HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer HotstoreMaxSpaceSafetyBuffer uint64 - - // Perform a full warmup from the coldstore to the hotstore upon splitstore startup. - // This is useful in the case you are migrating from a non-splitstore setup to splitstore. - // This should be false in the common case where a node is initialized from a snapshot - // since snapshots are loaded directly to the hotstore. - FullWarmup bool } // Full Node diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 2ff18f956e5..ac16a0f7e76 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -65,7 +65,6 @@ func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.Locked MarkSetType: cfg.Splitstore.MarkSetType, DiscardColdBlocks: cfg.Splitstore.ColdStoreType == "discard", UniversalColdBlocks: cfg.Splitstore.ColdStoreType == "universal", - FullWarmup: cfg.Splitstore.FullWarmup, HotStoreMessageRetention: cfg.Splitstore.HotStoreMessageRetention, HotStoreFullGCFrequency: cfg.Splitstore.HotStoreFullGCFrequency, From aa1584ce6c52619ecc0b915bc1a982659a763e4b Mon Sep 17 00:00:00 2001 From: zenground0 Date: Fri, 27 Dec 2024 16:22:37 +0530 Subject: [PATCH 17/20] make gen --- node/config/doc_gen.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 4f0068f6135..feab3d5b6e0 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1201,15 +1201,6 @@ exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceThreshold`, is set. Moving GC will not occur when total moving size exceeds HotstoreMaxSpaceTarget - HotstoreMaxSpaceSafetyBuffer`, }, - { - Name: "FullWarmup", - Type: "bool", - - Comment: `Perform a full warmup from the coldstore to the hotstore upon splitstore startup. -This is useful in the case you are migrating from a non-splitstore setup to splitstore. -This should be false in the common case where a node is initialized from a snapshot -since snapshots are loaded directly to the hotstore.`, - }, }, "StorageMiner": { { From 7349047de729e6add88eb4b907538ff4351403c8 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Fri, 27 Dec 2024 17:57:54 +0530 Subject: [PATCH 18/20] make docsgen-cli --- documentation/en/default-lotus-config.toml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 82ee6162284..41bc082da38 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -219,15 +219,6 @@ # env var: LOTUS_CHAINSTORE_SPLITSTORE_HOTSTOREMAXSPACESAFETYBUFFER #HotstoreMaxSpaceSafetyBuffer = 50000000000 - # Perform a full warmup from the coldstore to the hotstore upon splitstore startup. - # This is useful in the case you are migrating from a non-splitstore setup to splitstore. - # This should be false in the common case where a node is initialized from a snapshot - # since snapshots are loaded directly to the hotstore. - # - # type: bool - # env var: LOTUS_CHAINSTORE_SPLITSTORE_FULLWARMUP - #FullWarmup = false - [Fevm] # EnableEthRPC enables eth_ RPC methods. From 0640eddf843f523bf5921ed248b6d7f4cbd21185 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Fri, 27 Dec 2024 20:27:35 +0530 Subject: [PATCH 19/20] hotstore now memrepo and somehow it works for compaction! --- blockstore/mem.go | 9 +++++++++ blockstore/sync.go | 6 ++++++ node/repo/fsrepo.go | 17 +++++++++++------ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/blockstore/mem.go b/blockstore/mem.go index 8dbf4b719ad..6babbf508e9 100644 --- a/blockstore/mem.go +++ b/blockstore/mem.go @@ -107,3 +107,12 @@ func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) func (m MemBlockstore) HashOnRead(enabled bool) { // no-op } + +func (m MemBlockstore) ForEachKey(f func(c cid.Cid) error) error { + for _, b := range m { + if err := f(b.Cid()); err != nil { + return err + } + } + return nil +} diff --git a/blockstore/sync.go b/blockstore/sync.go index 4f0cf830ee6..37d50983006 100644 --- a/blockstore/sync.go +++ b/blockstore/sync.go @@ -81,3 +81,9 @@ func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error func (m *SyncBlockstore) HashOnRead(enabled bool) { // noop } + +func (m *SyncBlockstore) ForEachKey(f func(c cid.Cid) error) error { + m.mu.RLock() + defer m.mu.RUnlock() + return m.bs.ForEachKey(f) +} diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index f7f6652b099..090fe2ccfa3 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -436,8 +436,9 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain log.Error("invalid blockstore domain", domain) return nil, nil, ErrInvalidBlockstoreDomain } - var bs *badgerbs.Blockstore + var bs blockstore.Blockstore var err error + var close func() error if domain == UniversalBlockstore { fsr.bsOnce.Do(func() { path := fsr.join(filepath.Join(fsDatastore, "chain")) @@ -458,12 +459,14 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain opts.SyncWrites = false } - bs, err = badgerbs.Open(opts) + bbs, err := badgerbs.Open(opts) if err != nil { fsr.bsErr = err return } - fsr.bs = blockstore.WrapIDStore(bs) + close = bbs.Close + fsr.bs = blockstore.WrapIDStore(bbs) + bs = fsr.bs }) err = fsr.bsErr } else { @@ -485,17 +488,19 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain fsr.bsHotErr = err return } - bs, err = badgerbs.Open(opts) + bbs, err := badgerbs.Open(opts) if err != nil { fsr.bsHotErr = err return } - fsr.bsHot = bs + fsr.bsHot = bbs + close = bbs.Close + bs = fsr.bsHot }) err = fsr.bsHotErr } - return bs, bs.Close, err + return bs, close, err } func (fsr *fsLockedRepo) SplitstorePath() (string, error) { From 155d145b007160103059c7b1cfdd66157a7f36d8 Mon Sep 17 00:00:00 2001 From: zenground0 Date: Fri, 27 Dec 2024 20:39:43 +0530 Subject: [PATCH 20/20] lint --- node/repo/fsrepo.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 090fe2ccfa3..b5ff65b3920 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -438,7 +438,7 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain } var bs blockstore.Blockstore var err error - var close func() error + var closer func() error if domain == UniversalBlockstore { fsr.bsOnce.Do(func() { path := fsr.join(filepath.Join(fsDatastore, "chain")) @@ -464,7 +464,7 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain fsr.bsErr = err return } - close = bbs.Close + closer = bbs.Close fsr.bs = blockstore.WrapIDStore(bbs) bs = fsr.bs }) @@ -494,13 +494,13 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain return } fsr.bsHot = bbs - close = bbs.Close + closer = bbs.Close bs = fsr.bsHot }) err = fsr.bsHotErr } - return bs, close, err + return bs, closer, err } func (fsr *fsLockedRepo) SplitstorePath() (string, error) {