From 6a04e3978d021d4c27b246994144eb36402903c1 Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Mon, 14 Dec 2020 21:00:50 +0200 Subject: [PATCH] feat: State Diff data for fraud proofs (#118) * fix: update gomod and gofmt * feat: initial scaffold for getting data * feat: hook state manager to diff db * feat: mvp diffdb with in-memory map * feat: hook diffdb on NewBlockchain * fix: use eth.diffdb in the API instead of re-instantiating it * test(ovm-state-manager): ensure that diffdb is called properly * test(ovm-state-manager): only update diffdb for non eth_call calls * fix: handle error if no state diff found * fix(blockchain): directly call diffdb.GetDiff * fix(ethapi): export StatediffProof json data * feat(diffdb): save if a key was mutated or not * feat(ovm-state-manager): save if a key was mutated * feat(ethapi): expose if a storage key was mutated or not in the diff * misc: fmt / lint / fix tests * fix(ovm-state-manager): use proper log.Error format * StateDiff over Sqlite3 (#128) * feat(diffdb): switch to sqlite3 * perf(diffdb): use db txs to write less often * test(ovm-state-manager): cleanup diff db each time * fix(core): intiialize diff db with 256 item cache * ci: fix import order * fix(blockchain): close the diff db handler * feat(cmd): add CLI params for db cache size * feat(backend): use chaindata dir * test(ovm-state-manager): remove globals * chore: use `defer` in tests for tmp files * fix(ovm): set diff keys on rest of state mutating calls * feat(diffdb): set account diffs at special key * test: sorted diff equality Co-authored-by: Karl Floersch --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 9 + .../addressmanager/Lib_AddressManager.go | 1 - .../addressresolver/Lib_AddressResolver.go | 1 - .../OVM_CanonicalTransactionChain.go | 1 - .../executionmanager/OVM_ExecutionManager.go | 1 - core/blockchain.go | 41 ++- core/dao_test.go | 1 - core/state/statedb.go | 39 ++- core/vm/interface.go | 2 + core/vm/ovm_state_manager.go | 84 ++++++- core/vm/ovm_state_manager_test.go | 234 ++++++++++++++++++ diffdb/db.go | 166 +++++++++++++ diffdb/db_test.go | 51 ++++ eth/api_backend.go | 5 + eth/backend.go | 7 +- eth/config.go | 2 + go.mod | 12 +- go.sum | 56 +++++ internal/ethapi/api.go | 94 ++++++- internal/ethapi/backend.go | 3 + les/api_backend.go | 5 + 23 files changed, 786 insertions(+), 31 deletions(-) create mode 100644 core/vm/ovm_state_manager_test.go create mode 100644 diffdb/db.go create mode 100644 diffdb/db_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 21db44a4d715..27934d77d262 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -166,6 +166,7 @@ var ( utils.RollupEnableVerifierFlag, utils.RollupAddressManagerOwnerAddressFlag, utils.RollupStateDumpPathFlag, + utils.RollupDiffDbFlag, } rpcFlags = []cli.Flag{ diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 96da800a9ec6..6753e05ecc65 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -79,6 +79,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.RollupAddressManagerOwnerAddressFlag, utils.RollupEnableVerifierFlag, utils.RollupStateDumpPathFlag, + utils.RollupDiffDbFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 76ef38b72aaf..1f5f4020064d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -852,6 +852,12 @@ var ( Value: eth.DefaultConfig.Rollup.StateDumpPath, EnvVar: "ROLLUP_STATE_DUMP_PATH", } + RollupDiffDbFlag = cli.Uint64Flag{ + Name: "rollup.diffdbcache", + Usage: "Number of diffdb batch updates", + Value: eth.DefaultConfig.DiffDbCache, + EnvVar: "ROLLUP_DIFFDB_CACHE", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1617,6 +1623,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setEth1(ctx, &cfg.Rollup) setRollup(ctx, &cfg.Rollup) + if ctx.GlobalIsSet(RollupDiffDbFlag.Name) { + cfg.DiffDbCache = ctx.GlobalUint64(RollupDiffDbFlag.Name) + } if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) } diff --git a/contracts/addressmanager/Lib_AddressManager.go b/contracts/addressmanager/Lib_AddressManager.go index 782286f8913a..37f75ee533e8 100644 --- a/contracts/addressmanager/Lib_AddressManager.go +++ b/contracts/addressmanager/Lib_AddressManager.go @@ -455,4 +455,3 @@ func (_LibAddressManager *LibAddressManagerFilterer) ParseOwnershipTransferred(l } return event, nil } - diff --git a/contracts/addressresolver/Lib_AddressResolver.go b/contracts/addressresolver/Lib_AddressResolver.go index 9093078cc57c..8bb809f65921 100644 --- a/contracts/addressresolver/Lib_AddressResolver.go +++ b/contracts/addressresolver/Lib_AddressResolver.go @@ -214,4 +214,3 @@ func (_LibAddressResolver *LibAddressResolverSession) Resolve(_name string) (com func (_LibAddressResolver *LibAddressResolverCallerSession) Resolve(_name string) (common.Address, error) { return _LibAddressResolver.Contract.Resolve(&_LibAddressResolver.CallOpts, _name) } - diff --git a/contracts/canonicaltransactionchain/OVM_CanonicalTransactionChain.go b/contracts/canonicaltransactionchain/OVM_CanonicalTransactionChain.go index a412de5f7d39..337b1dcb77a9 100644 --- a/contracts/canonicaltransactionchain/OVM_CanonicalTransactionChain.go +++ b/contracts/canonicaltransactionchain/OVM_CanonicalTransactionChain.go @@ -961,4 +961,3 @@ func (_OVMCanonicalTransactionChain *OVMCanonicalTransactionChainFilterer) Parse } return event, nil } - diff --git a/contracts/executionmanager/OVM_ExecutionManager.go b/contracts/executionmanager/OVM_ExecutionManager.go index 7aaf77b4b97c..5f6800542908 100644 --- a/contracts/executionmanager/OVM_ExecutionManager.go +++ b/contracts/executionmanager/OVM_ExecutionManager.go @@ -214,4 +214,3 @@ func (_OVMExecutionManager *OVMExecutionManagerSession) GetMaxTransactionGasLimi func (_OVMExecutionManager *OVMExecutionManagerCallerSession) GetMaxTransactionGasLimit() (*big.Int, error) { return _OVMExecutionManager.Contract.GetMaxTransactionGasLimit(&_OVMExecutionManager.CallOpts) } - diff --git a/core/blockchain.go b/core/blockchain.go index a99dabc191ee..1bf49c87cb10 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -164,6 +165,8 @@ type BlockChain struct { txLookupCache *lru.Cache // Cache for the most recent transaction lookup data. futureBlocks *lru.Cache // future blocks are blocks added for later processing + diffdb state.DiffDB // Diff + quit chan struct{} // blockchain quit channel running int32 // running must be called atomically // procInterrupt must be atomically called @@ -181,6 +184,20 @@ type BlockChain struct { terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. } +func NewBlockChainWithDiffDb(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, path string, cache uint64) (*BlockChain, error) { + diff, err := diffdb.NewDiffDb(path, cache) + if err != nil { + return nil, err + } + bc, err := NewBlockChain(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve) + if err != nil { + return nil, err + } + bc.diffdb = diff + + return bc, nil +} + // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator and // Processor. @@ -348,7 +365,7 @@ func (bc *BlockChain) loadLastState() error { return bc.Reset() } // Make sure the state associated with the block is available - if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { + if _, err := state.NewWithDiffDb(currentBlock.Root(), bc.stateCache, bc.diffdb); err != nil { // Dangling block without a state associated, init from scratch log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) if err := bc.repair(¤tBlock); err != nil { @@ -410,7 +427,7 @@ func (bc *BlockChain) SetHead(head uint64) error { if newHeadBlock == nil { newHeadBlock = bc.genesisBlock } else { - if _, err := state.New(newHeadBlock.Root(), bc.stateCache); err != nil { + if _, err := state.NewWithDiffDb(newHeadBlock.Root(), bc.stateCache, bc.diffdb); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis newHeadBlock = bc.genesisBlock } @@ -526,6 +543,11 @@ func (bc *BlockChain) SetCurrentBlock(block *types.Block) { bc.currentBlock.Store(block) } +// GetDiff retrieves the diffdb's state diff keys for a block +func (bc *BlockChain) GetDiff(block *big.Int) (diffdb.Diff, error) { + return bc.diffdb.GetDiff(block) +} + // CurrentFastBlock retrieves the current fast-sync head block of the canonical // chain. The block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentFastBlock() *types.Block { @@ -549,7 +571,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.stateCache) + return state.NewWithDiffDb(root, bc.stateCache, bc.diffdb) } // StateCache returns the caching database underpinning the blockchain instance. @@ -601,7 +623,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { func (bc *BlockChain) repair(head **types.Block) error { for { // Abort if we've rewound to a head block that does have associated state - if _, err := state.New((*head).Root(), bc.stateCache); err == nil { + if _, err := state.NewWithDiffDb((*head).Root(), bc.stateCache, bc.diffdb); err == nil { log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash()) return nil } @@ -894,6 +916,15 @@ func (bc *BlockChain) Stop() { log.Error("Dangling trie nodes after full cleanup") } } + + if bc.diffdb != nil { + if err := bc.diffdb.ForceCommit(); err != nil { + log.Error("Failed to commit recent state diffs", "err", err) + } + if err := bc.diffdb.Close(); err != nil { + log.Error("Failed to commit state diffs handler", "err", err) + } + } log.Info("Blockchain manager stopped") } @@ -1683,7 +1714,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if parent == nil { parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1) } - statedb, err := state.New(parent.Root, bc.stateCache) + statedb, err := state.NewWithDiffDb(parent.Root, bc.stateCache, bc.diffdb) if err != nil { return it.index, err } diff --git a/core/dao_test.go b/core/dao_test.go index 4e8dba9e8488..50c13a6bf0ee 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -122,7 +122,6 @@ func TestDAOForkRangeExtradata(t *testing.T) { gspec.MustCommit(db) bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil) defer bc.Stop() - blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) for j := 0; j < len(blocks)/2; j++ { blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j] diff --git a/core/state/statedb.go b/core/state/statedb.go index 085f2379fbed..51a9997e7a59 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" @@ -57,15 +58,24 @@ func (n *proofList) Delete(key []byte) error { panic("not supported") } +// DiffDb is a database for storing state diffs per block +type DiffDB interface { + SetDiffKey(*big.Int, common.Address, common.Hash, bool) error + SetDiffAccount(*big.Int, common.Address) error + GetDiff(*big.Int) (diffdb.Diff, error) + Close() error + ForceCommit() error +} + // StateDBs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: // * Contracts // * Accounts type StateDB struct { - db Database - trie Trie - + db Database + trie Trie + diffdb DiffDB // This map holds 'live' objects, which will get modified while processing a state transition. stateObjects map[common.Address]*stateObject stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie @@ -123,6 +133,15 @@ func New(root common.Hash, db Database) (*StateDB, error) { }, nil } +func NewWithDiffDb(root common.Hash, db Database, diffdb DiffDB) (*StateDB, error) { + res, err := New(root, db) + if err != nil { + return nil, err + } + res.diffdb = diffdb + return res, nil +} + // setError remembers the first non-nil error it is called with. func (s *StateDB) setError(err error) { if s.dbErr == nil { @@ -130,6 +149,20 @@ func (s *StateDB) setError(err error) { } } +func (s *StateDB) SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error { + if s.diffdb == nil { + return errors.New("DiffDB not set") + } + return s.diffdb.SetDiffKey(block, address, key, mutated) +} + +func (s *StateDB) SetDiffAccount(block *big.Int, address common.Address) error { + if s.diffdb == nil { + return errors.New("DiffDB not set") + } + return s.diffdb.SetDiffAccount(block, address) +} + func (s *StateDB) Error() error { return s.dbErr } diff --git a/core/vm/interface.go b/core/vm/interface.go index dd401466adfa..e5a97141279b 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -64,6 +64,8 @@ type StateDB interface { AddPreimage(common.Hash, []byte) ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error + SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error + SetDiffAccount(block *big.Int, address common.Address) error } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/core/vm/ovm_state_manager.go b/core/vm/ovm_state_manager.go index 7eab29d2c767..c30c9e3729e7 100644 --- a/core/vm/ovm_state_manager.go +++ b/core/vm/ovm_state_manager.go @@ -23,10 +23,10 @@ var funcs = map[string]stateManagerFunction{ "hasAccount": nativeFunctionTrue, "hasEmptyAccount": nativeFunctionTrue, "hasContractStorage": nativeFunctionTrue, - "testAndSetAccountLoaded": nativeFunctionTrue, - "testAndSetAccountChanged": nativeFunctionTrue, - "testAndSetContractStorageLoaded": nativeFunctionTrue, - "testAndSetContractStorageChanged": nativeFunctionTrue, + "testAndSetAccountLoaded": testAndSetAccount, + "testAndSetAccountChanged": testAndSetAccount, + "testAndSetContractStorageLoaded": testAndSetContractStorageLoaded, + "testAndSetContractStorageChanged": testAndSetContractStorageChanged, "incrementTotalUncommittedAccounts": nativeFunctionVoid, "incrementTotalUncommittedContractStorage": nativeFunctionVoid, "initPendingAccount": nativeFunctionVoid, @@ -131,11 +131,85 @@ func putContractStorage(evm *EVM, contract *Contract, args map[string]interface{ return nil, errors.New("Could not parse value arg in putContractStorage") } val := toHash(_value) - evm.StateDB.SetState(address, key, val) + + // save the block number and address with modified key if it's not an eth_call + if evm.Context.EthCallSender == nil { + // save the value before + before := evm.StateDB.GetState(address, key) + evm.StateDB.SetState(address, key, val) + err := evm.StateDB.SetDiffKey( + evm.Context.BlockNumber, + address, + key, + before != val, + ) + if err != nil { + log.Error("Cannot set diff key", "err", err) + } + } else { + // otherwise just do the db update + evm.StateDB.SetState(address, key, val) + } + log.Debug("Put contract storage", "address", address.Hex(), "key", key.Hex(), "val", val.Hex()) return []interface{}{}, nil } +func testAndSetAccount(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) { + address, ok := args["_address"].(common.Address) + if !ok { + return nil, errors.New("Could not parse address arg in putContractStorage") + } + + if evm.Context.EthCallSender == nil { + err := evm.StateDB.SetDiffAccount( + evm.Context.BlockNumber, + address, + ) + + if err != nil { + log.Error("Cannot set account diff", err) + } + } + + return []interface{}{true}, nil +} + +func testAndSetContractStorageLoaded(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) { + return testAndSetContractStorage(evm, contract, args, false) +} + +func testAndSetContractStorageChanged(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) { + return testAndSetContractStorage(evm, contract, args, true) +} + +func testAndSetContractStorage(evm *EVM, contract *Contract, args map[string]interface{}, changed bool) ([]interface{}, error) { + address, ok := args["_contract"].(common.Address) + if !ok { + return nil, errors.New("Could not parse address arg in putContractStorage") + } + _key, ok := args["_key"] + if !ok { + return nil, errors.New("Could not parse key arg in putContractStorage") + } + key := toHash(_key) + + if evm.Context.EthCallSender == nil { + err := evm.StateDB.SetDiffKey( + evm.Context.BlockNumber, + address, + key, + changed, + ) + if err != nil { + log.Error("Cannot set diff key", "err", err) + } + } + + log.Debug("Test and Set Contract Storage", "address", address.Hex(), "key", key.Hex(), "changed", changed) + return []interface{}{true}, nil +} + func nativeFunctionTrue(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) { return []interface{}{true}, nil } diff --git a/core/vm/ovm_state_manager_test.go b/core/vm/ovm_state_manager_test.go new file mode 100644 index 000000000000..75c1cb95795f --- /dev/null +++ b/core/vm/ovm_state_manager_test.go @@ -0,0 +1,234 @@ +package vm + +import ( + "crypto/rand" + "math/big" + "os" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/diffdb" + "github.com/ethereum/go-ethereum/params" +) + +type TestData map[*big.Int]BlockData + +// per-block test data are an address + a bunch of k/v pairs +type BlockData map[common.Address][]ContractData + +// keys and values are bytes32 in solidity +type ContractData struct { + key [32]uint8 + value [32]uint8 + mutated bool +} + +// Test contract addrs +var ( + contract1 = common.HexToAddress("0x000000000000000000000000000000000001") + contract2 = common.HexToAddress("0x000000000000000000000000000000000002") +) + +func makeEnv(dbname string) (*diffdb.DiffDb, *EVM, TestData, *Contract) { + db, _ := diffdb.NewDiffDb(dbname, 1) + mock := &mockDb{db: *db} + env := NewEVM(Context{}, mock, params.TestChainConfig, Config{}) + // re-use `dummyContractRef` from `logger_test.go` + contract := NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 0) + testData := make(TestData) + return db, env, testData, contract +} + +func TestEthCallNoop(t *testing.T) { + db, env, _, contract := makeEnv("test1") + defer os.Remove("test1") + env.Context.EthCallSender = &common.Address{0} + env.Context.BlockNumber = big.NewInt(1) + args := map[string]interface{}{ + "_contract": contract1, + "_key": [32]uint8{1}, + "_value": [32]uint8{2}, + } + putContractStorage(env, contract, args) + diff, err := db.GetDiff(env.Context.BlockNumber) + if err != nil { + t.Fatal("Db call error", err) + } + if len(diff) > 0 { + t.Fatalf("map must be empty since it was an eth call") + } +} + +func TestSetDiffs(t *testing.T) { + db, env, testData, contract := makeEnv("test2") + defer os.Remove("test2") + // not an eth-call + env.Context.EthCallSender = nil + // in block 1 both contracts get touched + blockNumber := big.NewInt(5) + testData.addRandomData(blockNumber, contract1, 5) + testData.addRandomData(blockNumber, contract2, 10) + + // in another block, only 1 contract gets touched + blockNumber2 := big.NewInt(6) + testData.addRandomData(blockNumber2, contract2, 10) + + // insert the data in the diffdb via `putContractStorage` calls + putTestData(t, env, contract, blockNumber, testData) + + // diffs match + diff, _ := db.GetDiff(blockNumber) + expected := getExpected(testData[blockNumber]) + if !DiffsEqual(diff, expected) { + t.Fatalf("Diff did not match.") + } + + // empty diff for the next block + diff2, err := db.GetDiff(blockNumber2) + if err != nil { + t.Fatal("Db call error", err) + } + if len(diff2) != 0 { + t.Fatalf("Diff2 should be empty since data about the next block is not added yet") + } + + // insert the data and get the diff again + putTestData(t, env, contract, blockNumber2, testData) + + expected2 := getExpected(testData[blockNumber2]) + diff2, err = db.GetDiff(blockNumber2) + if err != nil { + t.Fatal("Db call error", err) + } + if !DiffsEqual(diff2, expected2) { + t.Fatalf("Diff did not match.") + } +} + +/// Sorted equality between 2 diffs +func DiffsEqual(d1 diffdb.Diff, d2 diffdb.Diff) bool { + for k, v := range d1 { + sort.SliceStable(v, func(i, j int) bool { + return v[i].Key.Big().Cmp(v[j].Key.Big()) < 0 + }) + + sort.SliceStable(d2[k], func(i, j int) bool { + return d2[k][i].Key.Big().Cmp(d2[k][j].Key.Big()) < 0 + }) + + exp := d2[k] + for i, v2 := range v { + if exp[i] != v2 { + return false + } + } + } + + return true +} + +// inserts a bunch of data for the provided `blockNumber` for all contracts touched in that block +func putTestData(t *testing.T, env *EVM, contract *Contract, blockNumber *big.Int, testData TestData) { + blockData := testData[blockNumber] + env.Context.BlockNumber = blockNumber + for address, data := range blockData { + for _, contractData := range data { + args := map[string]interface{}{ + "_contract": address, + "_key": contractData.key, + "_value": contractData.value, + } + _, err := putContractStorage(env, contract, args) + if err != nil { + t.Fatalf("Expected nil error, got %s", err) + } + } + } +} + +// creates `num` random k/v entries for `contract`'s address at `blockNumber` +func (data TestData) addRandomData(blockNumber *big.Int, contract common.Address, num int) { + for i := 0; i < num; i++ { + val := ContractData{ + key: randBytes(), + value: randBytes(), + mutated: true, + } + + // alloc empty blockdata + if data[blockNumber] == nil { + data[blockNumber] = make(BlockData) + } + data[blockNumber][contract] = append(data[blockNumber][contract], val) + } +} + +// the expected diff for the GetDiff call contains the data's keys only, the values & proofs +// are fetched via GetProof +func getExpected(testData BlockData) diffdb.Diff { + res := make(diffdb.Diff) + for address, data := range testData { + for _, contractData := range data { + key := diffdb.Key{ + Key: contractData.key, + Mutated: contractData.mutated, + } + res[address] = append(res[address], key) + } + } + return res +} + +// creates a random 32 byte array +func randBytes() [32]uint8 { + bytes := make([]uint8, 32) + rand.Read(bytes) + var res [32]uint8 + copy(res[:], bytes) + return res +} + +// Mock everything else +type mockDb struct { + db diffdb.DiffDb +} + +func (mock *mockDb) SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error { + mock.db.SetDiffKey(block, address, key, mutated) + return nil +} + +func (mock *mockDb) SetDiffAccount(block *big.Int, address common.Address) error { + // mock.db.SetDiffAccount(block, address) + return nil +} + +func (mock *mockDb) CreateAccount(common.Address) {} +func (mock *mockDb) SubBalance(common.Address, *big.Int) {} +func (mock *mockDb) AddBalance(common.Address, *big.Int) {} +func (mock *mockDb) GetBalance(common.Address) *big.Int { return big.NewInt(0) } +func (mock *mockDb) GetNonce(common.Address) uint64 { return 0 } +func (mock *mockDb) SetNonce(common.Address, uint64) {} +func (mock *mockDb) GetCodeHash(common.Address) common.Hash { return common.Hash{} } +func (mock *mockDb) GetCode(common.Address) []byte { return []byte{} } +func (mock *mockDb) SetCode(common.Address, []byte) {} +func (mock *mockDb) GetCodeSize(common.Address) int { return 0 } +func (mock *mockDb) AddRefund(uint64) {} +func (mock *mockDb) SubRefund(uint64) {} +func (mock *mockDb) GetRefund() uint64 { return 0 } +func (mock *mockDb) GetCommittedState(common.Address, common.Hash) common.Hash { return common.Hash{} } +func (mock *mockDb) GetState(common.Address, common.Hash) common.Hash { return common.Hash{} } +func (mock *mockDb) SetState(common.Address, common.Hash, common.Hash) {} +func (mock *mockDb) Suicide(common.Address) bool { return true } +func (mock *mockDb) HasSuicided(common.Address) bool { return true } +func (mock *mockDb) Exist(common.Address) bool { return true } +func (mock *mockDb) Empty(common.Address) bool { return true } +func (mock *mockDb) RevertToSnapshot(int) {} +func (mock *mockDb) Snapshot() int { return 0 } +func (mock *mockDb) AddLog(*types.Log) {} +func (mock *mockDb) AddPreimage(common.Hash, []byte) {} +func (mock *mockDb) ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error { + return nil +} diff --git a/diffdb/db.go b/diffdb/db.go new file mode 100644 index 000000000000..84f4e526517c --- /dev/null +++ b/diffdb/db.go @@ -0,0 +1,166 @@ +package diffdb + +import ( + "github.com/ethereum/go-ethereum/common" + _ "github.com/mattn/go-sqlite3" + + "database/sql" + "math/big" +) + +type Key struct { + Key common.Hash + Mutated bool +} + +type Diff map[common.Address][]Key + +/// A DiffDb is a thin wrapper around an Sqlite3 connection. +/// +/// Its purpose is to store and fetch the storage keys corresponding to an address that was +/// touched in a block. +type DiffDb struct { + db *sql.DB + tx *sql.Tx + stmt *sql.Stmt + cache uint64 + // We have a db-wide counter for the number of db calls made which we reset + // whenever it hits `cache`. + numCalls uint64 +} + +/// This key is used to mark that an account's state has been modified (e.g. nonce or balance) +/// and that an account proof is required. +var accountKey = common.HexToHash("0xDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + +var insertStatement = ` +INSERT INTO diffs + (block, address, key, mutated) + VALUES + ($1, $2, $3, $4) +ON CONFLICT DO NOTHING +` +var createStmt = ` +CREATE TABLE IF NOT EXISTS diffs ( + block INTEGER, + address STRING, + key STRING, + mutated BOOL, + PRIMARY KEY (block, address, key) +) +` +var selectStmt = ` +SELECT * from diffs WHERE block = $1 +` + +/// Inserts a new row to the sqlite with the provided diff data. +func (diff *DiffDb) SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error { + // add 1 more insertion to the transaction + _, err := diff.stmt.Exec(block.Uint64(), address, key, mutated) + if err != nil { + return err + } + + // increment number of calls + diff.numCalls += 1 + + // if we had enough calls, commit it + if diff.numCalls == diff.cache { + if err := diff.ForceCommit(); err != nil { + return err + } + } + + return nil +} + +/// Inserts a new row to the sqlite indicating that the account was modified in that block +/// at a pre-set key +func (diff *DiffDb) SetDiffAccount(block *big.Int, address common.Address) error { + return diff.SetDiffKey(block, address, accountKey, true) +} + +/// Commits a pending diffdb transaction +func (diff *DiffDb) ForceCommit() error { + if err := diff.tx.Commit(); err != nil { + return err + } + return diff.resetTx() +} + +/// Gets all the rows for the matching block and converts them to a Diff map. +func (diff *DiffDb) GetDiff(blockNum *big.Int) (Diff, error) { + // make the query + rows, err := diff.db.Query(selectStmt, blockNum.Uint64()) + if err != nil { + return nil, err + } + + // initialize our data + res := make(Diff) + var block uint64 + var address common.Address + var key common.Hash + var mutated bool + for rows.Next() { + // deserialize the line + err = rows.Scan(&block, &address, &key, &mutated) + if err != nil { + return nil, err + } + // add the data to the map + res[address] = append(res[address], Key{key, mutated}) + } + + return res, rows.Err() +} + +// Initializes the transaction which we will be using to commit data to the db +func (diff *DiffDb) resetTx() error { + // reset the number of calls made + diff.numCalls = 0 + + // start a new tx + tx, err := diff.db.Begin() + if err != nil { + return err + } + diff.tx = tx + + // the tx is about inserts + stmt, err := diff.tx.Prepare(insertStatement) + if err != nil { + return err + } + diff.stmt = stmt + + return nil +} + +func (diff *DiffDb) Close() error { + return diff.db.Close() +} + +/// Instantiates a new DiffDb using sqlite at `path`, with `cache` insertions +/// done in a transaction before it gets committed to the database. +func NewDiffDb(path string, cache uint64) (*DiffDb, error) { + // get a handle + db, err := sql.Open("sqlite3", path) + if err != nil { + return nil, err + } + + // create the table if it does not exist + _, err = db.Exec(createStmt) + if err != nil { + return nil, err + } + + diffdb := &DiffDb{db: db, cache: cache} + + // initialize the transaction + if err := diffdb.resetTx(); err != nil { + return nil, err + } + return diffdb, nil +} diff --git a/diffdb/db_test.go b/diffdb/db_test.go new file mode 100644 index 000000000000..8f83de8b83e6 --- /dev/null +++ b/diffdb/db_test.go @@ -0,0 +1,51 @@ +package diffdb + +import ( + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestDiffDb(t *testing.T) { + db, err := NewDiffDb("./test_diff.db", 3) + // cleanup (sqlite will create the file if it doesn't exist) + defer os.Remove("./test_diff.db") + if err != nil { + t.Fatal(err) + } + + hashes := []common.Hash{ + common.Hash{0x0}, + common.Hash{0x1}, + common.Hash{0x2}, + } + addr := common.Address{0x1} + db.SetDiffKey(big.NewInt(1), common.Address{0x1, 0x2}, common.Hash{0x12, 0x13}, false) + db.SetDiffKey(big.NewInt(1), addr, hashes[0], false) + db.SetDiffKey(big.NewInt(1), addr, hashes[1], false) + db.SetDiffKey(big.NewInt(1), addr, hashes[2], false) + db.SetDiffKey(big.NewInt(1), common.Address{0x2}, common.Hash{0x99}, false) + db.SetDiffKey(big.NewInt(2), common.Address{0x2}, common.Hash{0x98}, true) + // try overwriting, ON CONFLICT clause gets hit + err = db.SetDiffKey(big.NewInt(2), common.Address{0x2}, common.Hash{0x98}, false) + if err != nil { + t.Fatal("should be able to resolve conflict without error at the sql level") + } + + diff, err := db.GetDiff(big.NewInt(1)) + if err != nil { + t.Fatal("Did not expect error") + } + for i := range hashes { + if hashes[i] != diff[addr][i].Key { + t.Fatal("Did not match", hashes[i], "got", diff[addr][i].Key) + } + } + + diff, _ = db.GetDiff(big.NewInt(2)) + if diff[common.Address{0x2}][0].Mutated != true { + t.Fatalf("Did not match mutated") + } +} diff --git a/eth/api_backend.go b/eth/api_backend.go index 8343e7f23db3..d490543d2766 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" @@ -94,6 +95,10 @@ func (b *EthAPIBackend) CurrentBlock() *types.Block { return b.eth.blockchain.CurrentBlock() } +func (b *EthAPIBackend) GetDiff(block *big.Int) (diffdb.Diff, error) { + return b.eth.blockchain.GetDiff(block) +} + func (b *EthAPIBackend) SetHead(number uint64) { b.eth.protocolManager.downloader.Cancel() b.eth.blockchain.SetHead(number) diff --git a/eth/backend.go b/eth/backend.go index 2757c2487e09..dfc1189020d3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -188,7 +189,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve) + + // Save the diffdb under chaindata/diffdb + diffdbPath := filepath.Join(ctx.ResolvePath("chaindata"), "diffdb") + eth.blockchain, err = core.NewBlockChainWithDiffDb(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, diffdbPath, config.DiffDbCache) if err != nil { return nil, err } @@ -228,7 +232,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { gpoParams.Default = config.Miner.GasPrice } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) - return eth, nil } diff --git a/eth/config.go b/eth/config.go index 7e194594c22e..fc88930c1360 100644 --- a/eth/config.go +++ b/eth/config.go @@ -66,6 +66,7 @@ var DefaultConfig = Config{ TxIngestionEnable: false, StateDumpPath: "https://raw.githubusercontent.com/ethereum-optimism/regenesis/master/master.json", }, + DiffDbCache: 256, } func init() { @@ -122,6 +123,7 @@ type Config struct { DatabaseHandles int `toml:"-"` DatabaseCache int DatabaseFreezer string + DiffDbCache uint64 TrieCleanCache int TrieDirtyCache int diff --git a/go.mod b/go.mod index d30c8fe9c8c7..3593f2dd9eb0 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c github.com/golang/snappy v0.0.1 - github.com/google/go-cmp v0.3.1 // indirect github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 github.com/hashicorp/golang-lru v0.5.4 @@ -40,6 +39,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-colorable v0.1.0 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/mattn/go-sqlite3 v1.9.0 github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c @@ -57,12 +57,12 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 - golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 - golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 // indirect - golang.org/x/sync v0.0.0-20181108010431-42b317875d0f - golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd - golang.org/x/text v0.3.2 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 + golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f + golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/tools/gopls v0.5.3 // indirect google.golang.org/appengine v1.6.6 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9 diff --git a/go.sum b/go.sum index 00459394126e..5071b89c9e67 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,7 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= @@ -86,6 +87,9 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM= @@ -109,6 +113,7 @@ github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iK github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -167,11 +172,16 @@ github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeC github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff h1:+6NUiITWwE5q1KO6SAfUX918c+Tab0+tGAM/mtdlUyA= github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= @@ -192,38 +202,76 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U= golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201110032815-ae45d7cf37a9 h1:S0ATUNSw92bO7f5Z+1tSQsq48TdQs2Ye2rTGJdtf5qc= +golang.org/x/tools v0.0.0-20201110032815-ae45d7cf37a9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124005743-911501bfb504 h1:jOKV2ysikH1GANB7t2LotmhyvkkPvl7HQoEXkV6slJA= +golang.org/x/tools/gopls v0.5.3 h1:C8QSrqjqaVzlVoHL1R9yWbROoOApRgI8gN1G+cHlPuw= +golang.org/x/tools/gopls v0.5.3/go.mod h1:W4RRjThReIhl4bsed+iA7eJSLpDU8cpR3lWA3l6Khf4= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= @@ -239,5 +287,13 @@ gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHO gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc= +honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= +mvdan.cc/gofumpt v0.0.0-20200927160801-5bfeb2e70dd6 h1:z+/YqapuV7VZPvBb3GYmuEJbA88M3PFUxaHilHYVCpQ= +mvdan.cc/gofumpt v0.0.0-20200927160801-5bfeb2e70dd6/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws= +mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A= +mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0a1ecfae2577..64b9858edd9d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -549,9 +550,82 @@ type AccountResult struct { StorageProof []StorageResult `json:"storageProof"` } type StorageResult struct { - Key string `json:"key"` - Value *hexutil.Big `json:"value"` - Proof []string `json:"proof"` + Key string `json:"key"` + Value *hexutil.Big `json:"value"` + Proof []string `json:"proof"` + Mutated bool `json:"mutated"` +} + +// Result structs for GetStateDiffProof +type StateDiffProof struct { + Header *HeaderMeta `json:"header"` + Accounts []AccountResult `json:"accounts"` +} +type HeaderMeta struct { + Number *big.Int `json:"number"` + Hash common.Hash `json:"hash"` + StateRoot common.Hash `json:"stateRoot"` + Timestamp uint64 `json:"timestamp"` +} + +func (s *PublicBlockChainAPI) GetStateDiff(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (diffdb.Diff, error) { + _, header, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if err != nil { + return nil, err + } + return s.b.GetDiff(header.Number) +} + +// GetStateDiffProof returns the Merkle-proofs corresponding to all the accounts and +// storage slots which were touched for a given block number or hash. +func (s *PublicBlockChainAPI) GetStateDiffProof(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*StateDiffProof, error) { + state, header, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || header == nil || err != nil { + return nil, err + } + + // get the changed accounts for this block + diffs, err := s.GetStateDiff(ctx, blockNrOrHash) + if err != nil { + return nil, err + } + + // for each changed account, get their proof + var accounts []AccountResult + for address, keys := range diffs { + // need to convert the hashes to strings, we could maybe refactor getProof + // alternatively + keyStrings := make([]string, len(keys)) + for i, key := range keys { + keyStrings[i] = key.Key.String() + } + + // get the proofs + res, err := s.GetProof(ctx, address, keyStrings, blockNrOrHash) + if err != nil { + return nil, err + } + + // iterate over all the proofs and set their mutated bit + for i := range res.StorageProof { + res.StorageProof[i].Mutated = keys[i].Mutated + } + + accounts = append(accounts, *res) + } + + // add some metadata + stateDiffProof := &StateDiffProof{ + Header: &HeaderMeta{ + Number: header.Number, + Hash: header.Hash(), + StateRoot: header.Root, + Timestamp: header.Time, + }, + Accounts: accounts, + } + + return stateDiffProof, state.Error() } // GetProof returns the Merkle-proof for a given account and optionally some storage keys. @@ -581,9 +655,19 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre if storageError != nil { return nil, storageError } - storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)} + // by default, the GetProof API does not return if a storage item + // was mutated or not. + storageProof[i] = StorageResult{ + Key: key, + Value: (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), + Proof: common.ToHexArray(proof), + } } else { - storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + storageProof[i] = StorageResult{ + Key: key, + Value: &hexutil.Big{}, + Proof: []string{}, + } } } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 324f0cdc0250..8465b1ea9579 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -95,6 +96,8 @@ type Backend interface { ChainConfig() *params.ChainConfig CurrentBlock() *types.Block + + GetDiff(*big.Int) (diffdb.Diff, error) } func GetAPIs(apiBackend Backend) []rpc.API { diff --git a/les/api_backend.go b/les/api_backend.go index a6954dd8c841..8f3a730511f4 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/diffdb" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" @@ -81,6 +82,10 @@ func (b *LesApiBackend) CurrentBlock() *types.Block { return types.NewBlockWithHeader(b.eth.BlockChain().CurrentHeader()) } +func (b *LesApiBackend) GetDiff(*big.Int) (diffdb.Diff, error) { + return nil, errors.New("Diffs not supported in light client mode") +} + func (b *LesApiBackend) SetHead(number uint64) { b.eth.handler.downloader.Cancel() b.eth.blockchain.SetHead(number)