Skip to content

Commit 71705d0

Browse files
author
unclereal
committed
pipeline commit trie
1 parent 74f6b61 commit 71705d0

12 files changed

+139
-68
lines changed

core/block_validator.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
106106
// transition, such as amount of used gas, the receipt roots and the state root
107107
// itself. ValidateState returns a database batch if the validation was a success
108108
// otherwise nil and an error is returned.
109-
func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error {
109+
func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64, skipHeavyVerify bool) error {
110110
header := block.Header()
111111
if block.GasUsed() != usedGas {
112112
return fmt.Errorf("invalid gas used (remote: %d local: %d)", block.GasUsed(), usedGas)
@@ -129,13 +129,15 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
129129
return nil
130130
}
131131
},
132-
func() error {
132+
}
133+
if !skipHeavyVerify {
134+
validateFuns = append(validateFuns, func() error {
133135
if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
134136
return fmt.Errorf("invalid merkle root (remote: %x local: %x)", header.Root, root)
135137
} else {
136138
return nil
137139
}
138-
},
140+
})
139141
}
140142
validateRes := make(chan error, len(validateFuns))
141143
for _, f := range validateFuns {

core/blockchain.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -2073,7 +2073,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
20732073
// Validate the state using the default validator
20742074
substart = time.Now()
20752075
if !statedb.IsLightProcessed() {
2076-
if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil {
2076+
if err := bc.validator.ValidateState(block, statedb, receipts, usedGas, true); err != nil {
20772077
log.Error("validate state failed", "error", err)
20782078
bc.reportBlock(block, receipts, err)
20792079
return it.index, err

core/blockchain_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
156156
blockchain.reportBlock(block, receipts, err)
157157
return err
158158
}
159-
err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas)
159+
err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas, false)
160160
if err != nil {
161161
blockchain.reportBlock(block, receipts, err)
162162
return err

core/state/snapshot/difflayer.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ type diffLayer struct {
118118
storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
119119
storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted)
120120

121-
diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
121+
verifiedCh chan struct{}
122+
diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
122123

123124
lock sync.RWMutex
124125
}
@@ -168,7 +169,7 @@ func (h storageBloomHasher) Sum64() uint64 {
168169

169170
// newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
170171
// level persistent database or a hierarchical diff already.
171-
func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
172+
func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer {
172173
// Create the new layer with some pre-allocated data segments
173174
dl := &diffLayer{
174175
parent: parent,
@@ -177,6 +178,7 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s
177178
accountData: accounts,
178179
storageData: storage,
179180
storageList: make(map[common.Hash][]common.Hash),
181+
verifiedCh: verified,
180182
}
181183
switch parent := parent.(type) {
182184
case *diskLayer:
@@ -256,6 +258,15 @@ func (dl *diffLayer) Root() common.Hash {
256258
return dl.root
257259
}
258260

261+
// WaitVerified will wait until the diff layer been verified
262+
func (dl *diffLayer) WaitVerified() {
263+
if dl.verifiedCh == nil {
264+
return
265+
}
266+
<-dl.verifiedCh
267+
return
268+
}
269+
259270
// Parent returns the subsequent layer of a diff layer.
260271
func (dl *diffLayer) Parent() snapshot {
261272
return dl.parent
@@ -423,8 +434,8 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([
423434

424435
// Update creates a new layer on top of the existing snapshot diff tree with
425436
// the specified data items.
426-
func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
427-
return newDiffLayer(dl, blockRoot, destructs, accounts, storage)
437+
func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer {
438+
return newDiffLayer(dl, blockRoot, destructs, accounts, storage, verified)
428439
}
429440

430441
// flatten pushes all data from this point downwards, flattening everything into

core/state/snapshot/difflayer_test.go

+13-13
Original file line numberDiff line numberDiff line change
@@ -151,13 +151,13 @@ func TestMergeDelete(t *testing.T) {
151151
}
152152
}
153153
// Add some flipAccs-flopping layers on top
154-
parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage)
155-
child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
156-
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
157-
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
158-
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
159-
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
160-
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
154+
parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage, nil)
155+
child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage, nil)
156+
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage, nil)
157+
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage, nil)
158+
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage, nil)
159+
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage, nil)
160+
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage, nil)
161161

162162
if data, _ := child.Account(h1); data == nil {
163163
t.Errorf("last diff layer: expected %x account to be non-nil", h1)
@@ -209,7 +209,7 @@ func TestInsertAndMerge(t *testing.T) {
209209
accounts = make(map[common.Hash][]byte)
210210
storage = make(map[common.Hash]map[common.Hash][]byte)
211211
)
212-
parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage)
212+
parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage, nil)
213213
}
214214
{
215215
var (
@@ -220,7 +220,7 @@ func TestInsertAndMerge(t *testing.T) {
220220
accounts[acc] = randomAccount()
221221
storage[acc] = make(map[common.Hash][]byte)
222222
storage[acc][slot] = []byte{0x01}
223-
child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
223+
child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage, nil)
224224
}
225225
// And flatten
226226
merged := (child.flatten()).(*diffLayer)
@@ -256,7 +256,7 @@ func BenchmarkSearch(b *testing.B) {
256256
for i := 0; i < 10000; i++ {
257257
accounts[randomHash()] = randomAccount()
258258
}
259-
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
259+
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage, nil)
260260
}
261261
var layer snapshot
262262
layer = emptyLayer()
@@ -298,7 +298,7 @@ func BenchmarkSearchSlot(b *testing.B) {
298298
accStorage[randomHash()] = value
299299
storage[accountKey] = accStorage
300300
}
301-
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
301+
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage, nil)
302302
}
303303
var layer snapshot
304304
layer = emptyLayer()
@@ -336,7 +336,7 @@ func BenchmarkFlatten(b *testing.B) {
336336
}
337337
storage[accountKey] = accStorage
338338
}
339-
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
339+
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage, nil)
340340
}
341341
b.ResetTimer()
342342
for i := 0; i < b.N; i++ {
@@ -386,7 +386,7 @@ func BenchmarkJournal(b *testing.B) {
386386
}
387387
storage[accountKey] = accStorage
388388
}
389-
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
389+
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage, nil)
390390
}
391391
layer := snapshot(new(diskLayer))
392392
for i := 1; i < 128; i++ {

core/state/snapshot/disklayer.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ func (dl *diskLayer) Root() common.Hash {
4949
return dl.root
5050
}
5151

52+
func (dl *diskLayer) WaitVerified() {
53+
return
54+
}
55+
5256
// Parent always returns nil as there's no layer below the disk.
5357
func (dl *diskLayer) Parent() snapshot {
5458
return nil
@@ -161,6 +165,6 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
161165
// Update creates a new layer on top of the existing snapshot diff tree with
162166
// the specified data items. Note, the maps are retained by the method to avoid
163167
// copying everything.
164-
func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer {
165-
return newDiffLayer(dl, blockHash, destructs, accounts, storage)
168+
func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer {
169+
return newDiffLayer(dl, blockHash, destructs, accounts, storage, verified)
166170
}

core/state/snapshot/iterator_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func TestAccountIteratorBasics(t *testing.T) {
5353
}
5454
}
5555
// Add some (identical) layers on top
56-
diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
56+
diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage), nil)
5757
it := diffLayer.AccountIterator(common.Hash{})
5858
verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator
5959

@@ -91,7 +91,7 @@ func TestStorageIteratorBasics(t *testing.T) {
9191
nilStorage[h] = nilstorage
9292
}
9393
// Add some (identical) layers on top
94-
diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage))
94+
diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage), nil)
9595
for account := range accounts {
9696
it, _ := diffLayer.StorageIterator(account, common.Hash{})
9797
verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator

core/state/snapshot/journal.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
243243
}
244244
storageData[entry.Hash] = slots
245245
}
246-
return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r)
246+
return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData, nil), r)
247247
}
248248

249249
// Journal terminates any in-progress snapshot generation, also implicitly pushing

core/state/snapshot/snapshot.go

+16-5
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ type Snapshot interface {
101101
// Root returns the root hash for which this snapshot was made.
102102
Root() common.Hash
103103

104+
// WaitVerified will wait until the Snapshot been verified
105+
WaitVerified()
106+
104107
// Account directly retrieves the account associated with a particular hash in
105108
// the snapshot slim data format.
106109
Account(hash common.Hash) (*Account, error)
@@ -130,7 +133,7 @@ type snapshot interface {
130133
// the specified data items.
131134
//
132135
// Note, the maps are retained by the method to avoid copying everything.
133-
Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer
136+
Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer
134137

135138
// Journal commits an entire diff hierarchy to disk into a single journal entry.
136139
// This is meant to be used during shutdown to persist the snapshot without
@@ -289,6 +292,14 @@ func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot {
289292
return t.layers[blockRoot]
290293
}
291294

295+
// Snapshot retrieves a snapshot belonging to the given block root, or nil if no
296+
func (t *Tree) Remove(blockRoot common.Hash) {
297+
t.lock.RLock()
298+
defer t.lock.RUnlock()
299+
300+
delete(t.layers, blockRoot)
301+
}
302+
292303
// Snapshots returns all visited layers from the topmost layer with specific
293304
// root and traverses downward. The layer amount is limited by the given number.
294305
// If nodisk is set, then disk layer is excluded.
@@ -322,14 +333,14 @@ func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot {
322333
return ret
323334
}
324335

325-
func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Address]struct{}, accounts map[common.Address][]byte, storage map[common.Address]map[string][]byte) error {
336+
func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Address]struct{}, accounts map[common.Address][]byte, storage map[common.Address]map[string][]byte, verified chan struct{}) error {
326337
hashDestructs, hashAccounts, hashStorage := transformSnapData(destructs, accounts, storage)
327-
return t.update(blockRoot, parentRoot, hashDestructs, hashAccounts, hashStorage)
338+
return t.update(blockRoot, parentRoot, hashDestructs, hashAccounts, hashStorage, verified)
328339
}
329340

330341
// Update adds a new snapshot into the tree, if that can be linked to an existing
331342
// old parent. It is disallowed to insert a disk layer (the origin of all).
332-
func (t *Tree) update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {
343+
func (t *Tree) update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) error {
333344
// Reject noop updates to avoid self-loops in the snapshot tree. This is a
334345
// special case that can only happen for Clique networks where empty blocks
335346
// don't modify the state (0 block subsidy).
@@ -344,7 +355,7 @@ func (t *Tree) update(blockRoot common.Hash, parentRoot common.Hash, destructs m
344355
if parent == nil {
345356
return fmt.Errorf("parent [%#x] snapshot missing", parentRoot)
346357
}
347-
snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage)
358+
snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage, verified)
348359

349360
// Save the new snapshot for later
350361
t.lock.Lock()

0 commit comments

Comments
 (0)