Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Caplin: indexing to use right buf size #9998

Merged
merged 2 commits into from
Apr 21, 2024
Merged

Caplin: indexing to use right buf size #9998

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3476f4
save
AskAlexSharov Apr 19, 2024
afad432
save
AskAlexSharov Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions turbo/snapshotsync/freezeblocks/block_snapshots.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2100,7 +2100,7 @@ func HeadersIdx(ctx context.Context, info snaptype.FileInfo, salt uint32, tmpDir
}

func BodiesIdx(ctx context.Context, info snaptype.FileInfo, salt uint32, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
num := make([]byte, 8)
num := make([]byte, binary.MaxVarintLen64)

if err := Idx(ctx, info, salt, info.From, tmpDir, log.LvlDebug, p, func(idx *recsplit.RecSplit, i, offset uint64, _ []byte) error {
if p != nil {
Expand All @@ -2126,11 +2126,9 @@ func Idx(ctx context.Context, info snaptype.FileInfo, salt uint32, firstDataID u
}()

d, err := seg.NewDecompressor(info.Path)

if err != nil {
return fmt.Errorf("can't open %s for indexing: %w", info.Name(), err)
}

defer d.Close()

if p != nil {
Expand Down
6 changes: 3 additions & 3 deletions turbo/snapshotsync/freezeblocks/caplin_snapshots.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ import (
var sidecarSSZSize = (&cltypes.BlobSidecar{}).EncodingSizeSSZ()

func BeaconSimpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
num := make([]byte, binary.MaxVarintLen64)
if err := Idx(ctx, sn, salt, sn.From, tmpDir, log.LvlDebug, p, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
if i%20_000 == 0 {
logger.Log(lvl, fmt.Sprintf("Generating idx for %s", sn.Type.String()), "progress", i)
}
p.Processed.Add(1)
num := make([]byte, 8)
n := binary.PutUvarint(num, i)
if err := idx.AddKey(num[:n], offset); err != nil {
return err
Expand Down Expand Up @@ -484,7 +484,7 @@ func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64,
return nil
}

func DumpBlobsSidecar(ctx context.Context, blobStorage blob_storage.BlobStorage, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error {
func DumpBlobsSidecar(ctx context.Context, blobStorage blob_storage.BlobStorage, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, compressWorkers int, lvl log.Lvl, logger log.Logger) error {
for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, nil) {
blocksPerFile := snapcfg.MergeLimit("", i)

Expand All @@ -493,7 +493,7 @@ func DumpBlobsSidecar(ctx context.Context, blobStorage blob_storage.BlobStorage,
}
to := chooseSegmentEnd(i, toSlot, nil)
logger.Log(lvl, "Dumping blobs sidecars", "from", i, "to", to)
if err := dumpBlobSidecarsRange(ctx, db, blobStorage, i, to, salt, dirs, workers, lvl, logger); err != nil {
if err := dumpBlobSidecarsRange(ctx, db, blobStorage, i, to, salt, dirs, compressWorkers, lvl, logger); err != nil {
return err
}
}
Expand Down
Loading