feat: State Diff data for fraud proofs (ethereum#118)

* fix: update gomod and gofmt

* feat: initial scaffold for getting data

* feat: hook state manager to diff db

* feat: mvp diffdb with in-memory map

* feat: hook diffdb on NewBlockchain

* fix: use eth.diffdb in the API instead of re-instantiating it

* test(ovm-state-manager): ensure that diffdb is called properly

* test(ovm-state-manager): only update diffdb for non eth_call calls

* fix: handle error if no state diff found

* fix(blockchain): directly call diffdb.GetDiff

* fix(ethapi): export StatediffProof json data

* feat(diffdb): save if a key was mutated or not

* feat(ovm-state-manager): save if a key was mutated

* feat(ethapi): expose if a storage key was mutated or not in the diff

* misc: fmt / lint / fix tests

* fix(ovm-state-manager): use proper log.Error format

* StateDiff over Sqlite3 (ethereum#128)

* feat(diffdb): switch to sqlite3

* perf(diffdb): use db txs to write less often

* test(ovm-state-manager): cleanup diff db each time

* fix(core): intiialize diff db with 256 item cache

* ci: fix import order

* fix(blockchain): close the diff db handler

* feat(cmd): add CLI params for db cache size

* feat(backend): use chaindata dir

* test(ovm-state-manager): remove globals

* chore: use `defer` in tests for tmp files

* fix(ovm): set diff keys on rest of state mutating calls

* feat(diffdb): set account diffs at special key

* test: sorted diff equality

Co-authored-by: Karl Floersch <karl@karlfloersch.com>

cmd/geth/main.go

@@ -166,6 +166,7 @@ var (
 		utils.RollupEnableVerifierFlag,
 		utils.RollupAddressManagerOwnerAddressFlag,
 		utils.RollupStateDumpPathFlag,
+		utils.RollupDiffDbFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/usage.go

@@ -79,6 +79,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.RollupAddressManagerOwnerAddressFlag,
 			utils.RollupEnableVerifierFlag,
 			utils.RollupStateDumpPathFlag,
+			utils.RollupDiffDbFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -852,6 +852,12 @@ var (
 		Value:  eth.DefaultConfig.Rollup.StateDumpPath,
 		EnvVar: "ROLLUP_STATE_DUMP_PATH",
 	}
+	RollupDiffDbFlag = cli.Uint64Flag{
+		Name:   "rollup.diffdbcache",
+		Usage:  "Number of diffdb batch updates",
+		Value:  eth.DefaultConfig.DiffDbCache,
+		EnvVar: "ROLLUP_DIFFDB_CACHE",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1617,6 +1623,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	setEth1(ctx, &cfg.Rollup)
 	setRollup(ctx, &cfg.Rollup)
 
+	if ctx.GlobalIsSet(RollupDiffDbFlag.Name) {
+		cfg.DiffDbCache = ctx.GlobalUint64(RollupDiffDbFlag.Name)
+	}
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
 	}

core/blockchain.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/diffdb"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
@@ -164,6 +165,8 @@ type BlockChain struct {
 	txLookupCache *lru.Cache     // Cache for the most recent transaction lookup data.
 	futureBlocks  *lru.Cache     // future blocks are blocks added for later processing
 
+	diffdb state.DiffDB // Diff
+
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
 	// procInterrupt must be atomically called
@@ -181,6 +184,20 @@ type BlockChain struct {
 	terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion.
 }
 
+func NewBlockChainWithDiffDb(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool, path string, cache uint64) (*BlockChain, error) {
+	diff, err := diffdb.NewDiffDb(path, cache)
+	if err != nil {
+		return nil, err
+	}
+	bc, err := NewBlockChain(db, cacheConfig, chainConfig, engine, vmConfig, shouldPreserve)
+	if err != nil {
+		return nil, err
+	}
+	bc.diffdb = diff
+
+	return bc, nil
+}
+
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialises the default Ethereum Validator and
 // Processor.
@@ -348,7 +365,7 @@ func (bc *BlockChain) loadLastState() error {
 		return bc.Reset()
 	}
 	// Make sure the state associated with the block is available
-	if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
+	if _, err := state.NewWithDiffDb(currentBlock.Root(), bc.stateCache, bc.diffdb); err != nil {
 		// Dangling block without a state associated, init from scratch
 		log.Warn("Head state missing, repairing chain", "number", currentBlock.Number(), "hash", currentBlock.Hash())
 		if err := bc.repair(&currentBlock); err != nil {
@@ -410,7 +427,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 			if newHeadBlock == nil {
 				newHeadBlock = bc.genesisBlock
 			} else {
-				if _, err := state.New(newHeadBlock.Root(), bc.stateCache); err != nil {
+				if _, err := state.NewWithDiffDb(newHeadBlock.Root(), bc.stateCache, bc.diffdb); err != nil {
 					// Rewound state missing, rolled back to before pivot, reset to genesis
 					newHeadBlock = bc.genesisBlock
 				}
@@ -526,6 +543,11 @@ func (bc *BlockChain) SetCurrentBlock(block *types.Block) {
 	bc.currentBlock.Store(block)
 }
 
+// GetDiff retrieves the diffdb's state diff keys for a block
+func (bc *BlockChain) GetDiff(block *big.Int) (diffdb.Diff, error) {
+	return bc.diffdb.GetDiff(block)
+}
+
 // CurrentFastBlock retrieves the current fast-sync head block of the canonical
 // chain. The block is retrieved from the blockchain's internal cache.
 func (bc *BlockChain) CurrentFastBlock() *types.Block {
@@ -549,7 +571,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) {
 
 // StateAt returns a new mutable state based on a particular point in time.
 func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
-	return state.New(root, bc.stateCache)
+	return state.NewWithDiffDb(root, bc.stateCache, bc.diffdb)
 }
 
 // StateCache returns the caching database underpinning the blockchain instance.
@@ -601,7 +623,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 func (bc *BlockChain) repair(head **types.Block) error {
 	for {
 		// Abort if we've rewound to a head block that does have associated state
-		if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
+		if _, err := state.NewWithDiffDb((*head).Root(), bc.stateCache, bc.diffdb); err == nil {
 			log.Info("Rewound blockchain to past state", "number", (*head).Number(), "hash", (*head).Hash())
 			return nil
 		}
@@ -894,6 +916,15 @@ func (bc *BlockChain) Stop() {
 			log.Error("Dangling trie nodes after full cleanup")
 		}
 	}
+
+	if bc.diffdb != nil {
+		if err := bc.diffdb.ForceCommit(); err != nil {
+			log.Error("Failed to commit recent state diffs", "err", err)
+		}
+		if err := bc.diffdb.Close(); err != nil {
+			log.Error("Failed to commit state diffs handler", "err", err)
+		}
+	}
 	log.Info("Blockchain manager stopped")
 }
 
@@ -1683,7 +1714,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 		if parent == nil {
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
-		statedb, err := state.New(parent.Root, bc.stateCache)
+		statedb, err := state.NewWithDiffDb(parent.Root, bc.stateCache, bc.diffdb)
 		if err != nil {
 			return it.index, err
 		}

core/dao_test.go

@@ -122,7 +122,6 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	gspec.MustCommit(db)
 	bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer bc.Stop()
-
 	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
 	for j := 0; j < len(blocks)/2; j++ {
 		blocks[j], blocks[len(blocks)-1-j] = blocks[len(blocks)-1-j], blocks[j]

core/state/statedb.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/diffdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -57,15 +58,24 @@ func (n *proofList) Delete(key []byte) error {
 	panic("not supported")
 }
 
+// DiffDb is a database for storing state diffs per block
+type DiffDB interface {
+	SetDiffKey(*big.Int, common.Address, common.Hash, bool) error
+	SetDiffAccount(*big.Int, common.Address) error
+	GetDiff(*big.Int) (diffdb.Diff, error)
+	Close() error
+	ForceCommit() error
+}
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
 // * Contracts
 // * Accounts
 type StateDB struct {
-	db   Database
-	trie Trie
-
+	db     Database
+	trie   Trie
+	diffdb DiffDB
 	// This map holds 'live' objects, which will get modified while processing a state transition.
 	stateObjects        map[common.Address]*stateObject
 	stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie
@@ -123,13 +133,36 @@ func New(root common.Hash, db Database) (*StateDB, error) {
 	}, nil
 }
 
+func NewWithDiffDb(root common.Hash, db Database, diffdb DiffDB) (*StateDB, error) {
+	res, err := New(root, db)
+	if err != nil {
+		return nil, err
+	}
+	res.diffdb = diffdb
+	return res, nil
+}
+
 // setError remembers the first non-nil error it is called with.
 func (s *StateDB) setError(err error) {
 	if s.dbErr == nil {
 		s.dbErr = err
 	}
 }
 
+func (s *StateDB) SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error {
+	if s.diffdb == nil {
+		return errors.New("DiffDB not set")
+	}
+	return s.diffdb.SetDiffKey(block, address, key, mutated)
+}
+
+func (s *StateDB) SetDiffAccount(block *big.Int, address common.Address) error {
+	if s.diffdb == nil {
+		return errors.New("DiffDB not set")
+	}
+	return s.diffdb.SetDiffAccount(block, address)
+}
+
 func (s *StateDB) Error() error {
 	return s.dbErr
 }

core/vm/interface.go

@@ -64,6 +64,8 @@ type StateDB interface {
 	AddPreimage(common.Hash, []byte)
 
 	ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error
+	SetDiffKey(block *big.Int, address common.Address, key common.Hash, mutated bool) error
+	SetDiffAccount(block *big.Int, address common.Address) error
 }
 
 // CallContext provides a basic interface for the EVM calling conventions. The EVM

core/vm/ovm_state_manager.go

@@ -23,10 +23,10 @@ var funcs = map[string]stateManagerFunction{
 	"hasAccount":                               nativeFunctionTrue,
 	"hasEmptyAccount":                          nativeFunctionTrue,
 	"hasContractStorage":                       nativeFunctionTrue,
-	"testAndSetAccountLoaded":                  nativeFunctionTrue,
-	"testAndSetAccountChanged":                 nativeFunctionTrue,
-	"testAndSetContractStorageLoaded":          nativeFunctionTrue,
-	"testAndSetContractStorageChanged":         nativeFunctionTrue,
+	"testAndSetAccountLoaded":                  testAndSetAccount,
+	"testAndSetAccountChanged":                 testAndSetAccount,
+	"testAndSetContractStorageLoaded":          testAndSetContractStorageLoaded,
+	"testAndSetContractStorageChanged":         testAndSetContractStorageChanged,
 	"incrementTotalUncommittedAccounts":        nativeFunctionVoid,
 	"incrementTotalUncommittedContractStorage": nativeFunctionVoid,
 	"initPendingAccount":                       nativeFunctionVoid,
@@ -131,11 +131,85 @@ func putContractStorage(evm *EVM, contract *Contract, args map[string]interface{
 		return nil, errors.New("Could not parse value arg in putContractStorage")
 	}
 	val := toHash(_value)
-	evm.StateDB.SetState(address, key, val)
+
+	// save the block number and address with modified key if it's not an eth_call
+	if evm.Context.EthCallSender == nil {
+		// save the value before
+		before := evm.StateDB.GetState(address, key)
+		evm.StateDB.SetState(address, key, val)
+		err := evm.StateDB.SetDiffKey(
+			evm.Context.BlockNumber,
+			address,
+			key,
+			before != val,
+		)
+		if err != nil {
+			log.Error("Cannot set diff key", "err", err)
+		}
+	} else {
+		// otherwise just do the db update
+		evm.StateDB.SetState(address, key, val)
+	}
+
 	log.Debug("Put contract storage", "address", address.Hex(), "key", key.Hex(), "val", val.Hex())
 	return []interface{}{}, nil
 }
 
+func testAndSetAccount(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) {
+	address, ok := args["_address"].(common.Address)
+	if !ok {
+		return nil, errors.New("Could not parse address arg in putContractStorage")
+	}
+
+	if evm.Context.EthCallSender == nil {
+		err := evm.StateDB.SetDiffAccount(
+			evm.Context.BlockNumber,
+			address,
+		)
+
+		if err != nil {
+			log.Error("Cannot set account diff", err)
+		}
+	}
+
+	return []interface{}{true}, nil
+}
+
+func testAndSetContractStorageLoaded(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) {
+	return testAndSetContractStorage(evm, contract, args, false)
+}
+
+func testAndSetContractStorageChanged(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) {
+	return testAndSetContractStorage(evm, contract, args, true)
+}
+
+func testAndSetContractStorage(evm *EVM, contract *Contract, args map[string]interface{}, changed bool) ([]interface{}, error) {
+	address, ok := args["_contract"].(common.Address)
+	if !ok {
+		return nil, errors.New("Could not parse address arg in putContractStorage")
+	}
+	_key, ok := args["_key"]
+	if !ok {
+		return nil, errors.New("Could not parse key arg in putContractStorage")
+	}
+	key := toHash(_key)
+
+	if evm.Context.EthCallSender == nil {
+		err := evm.StateDB.SetDiffKey(
+			evm.Context.BlockNumber,
+			address,
+			key,
+			changed,
+		)
+		if err != nil {
+			log.Error("Cannot set diff key", "err", err)
+		}
+	}
+
+	log.Debug("Test and Set Contract Storage", "address", address.Hex(), "key", key.Hex(), "changed", changed)
+	return []interface{}{true}, nil
+}
+
 func nativeFunctionTrue(evm *EVM, contract *Contract, args map[string]interface{}) ([]interface{}, error) {
 	return []interface{}{true}, nil
 }