diff --git a/cmd/geth/config.go b/cmd/geth/config.go index dceda022629b..8d5f2ad17f33 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -159,7 +159,22 @@ func makeFullNode(ctx *cli.Context) *node.Node { utils.RegisterEthService(stack, &cfg.Eth) if ctx.GlobalBool(utils.StateDiffFlag.Name) { - utils.RegisterStateDiffService(stack) + var dbParams *[3]string + if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) { + dbParams = new([3]string) + dbParams[0] = ctx.GlobalString(utils.StateDiffDBFlag.Name) + if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) { + dbParams[1] = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name) + } else { + utils.Fatalf("Must specify node ID for statediff DB output") + } + if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) { + dbParams[2] = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name) + } else { + utils.Fatalf("Must specify client name for statediff DB output") + } + } + utils.RegisterStateDiffService(stack, dbParams) } // Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f2d142f4c5d3..a34cf01904b7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -147,6 +147,9 @@ var ( utils.EWASMInterpreterFlag, utils.EVMInterpreterFlag, utils.StateDiffFlag, + utils.StateDiffDBFlag, + utils.StateDiffDBNodeIDFlag, + utils.StateDiffDBClientNameFlag, configFileFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index ec2e03d62eaa..28dd448612a6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -255,6 +255,9 @@ var AppHelpFlagGroups = []flagGroup{ Name: "STATE DIFF", Flags: []cli.Flag{ utils.StateDiffFlag, + utils.StateDiffDBFlag, + utils.StateDiffDBNodeIDFlag, + utils.StateDiffDBClientNameFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8095b75078af..4efbfa1977a5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -761,6 +761,18 @@ var ( Name: "statediff", Usage: "Enables the processing of state diffs between each block", } + StateDiffDBFlag = cli.StringFlag{ + Name: "statediff.db", + Usage: "Postgres database connection string for writing state diffs", + } + StateDiffDBNodeIDFlag = cli.StringFlag{ + Name: "statediff.dbnodeid", + Usage: "Node ID to use when writing state diffs to database", + } + StateDiffDBClientNameFlag = cli.StringFlag{ + Name: "statediff.dbclientname", + Usage: "Client name to use when writing state diffs to database", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1633,12 +1645,15 @@ func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []st } // RegisterStateDiffService configures and registers a service to stream state diff data over RPC -func RegisterStateDiffService(stack *node.Node) { +// dbParams are: Postgres connection URI, Node ID, client name +func RegisterStateDiffService(stack *node.Node, dbParams *[3]string) { if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { var ethServ *eth.Ethereum - ctx.Service(ðServ) - blockChain := ethServ.BlockChain() - return statediff.NewStateDiffService(blockChain) + err := ctx.Service(ðServ) + if err != nil { + return nil, err + } + return statediff.NewStateDiffService(ethServ, dbParams) }); err != nil { Fatalf("Failed to register State Diff Service", err) } diff --git a/go.mod b/go.mod index b496b871ba9e..df3dbd35998c 100644 --- a/go.mod +++ b/go.mod @@ -21,36 +21,45 @@ require ( github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa - github.com/fatih/color v1.3.0 + github.com/fatih/color v1.7.0 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 - github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c + github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.1 github.com/google/go-cmp v0.3.1 // indirect - github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 + github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 - github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad + github.com/hashicorp/golang-lru v0.5.4 github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 + github.com/ipfs/go-cid v0.0.7 + github.com/ipfs/go-ipfs-blockstore v1.0.1 + github.com/ipfs/go-ipfs-ds-help v1.0.0 + github.com/ipfs/go-ipld-format v0.2.0 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 - github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 + github.com/jmoiron/sqlx v1.2.0 + github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 - github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.0 - github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/lib/pq v1.8.0 + github.com/mattn/go-colorable v0.1.1 + github.com/mattn/go-isatty v0.0.5 + github.com/multiformats/go-multihash v0.0.14 github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c + github.com/onsi/gomega v1.4.3 github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 - github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 + github.com/prometheus/tsdb v0.7.1 github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect + github.com/sirupsen/logrus v1.7.0 + github.com/spf13/viper v1.7.1 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect @@ -58,10 +67,10 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 - golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 + golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sync v0.0.0-20181108010431-42b317875d0f - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index f3ddb7ba0992..d4b813f87722 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,16 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -20,6 +33,7 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -33,9 +47,15 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= @@ -46,8 +66,14 @@ github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/ github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -71,50 +97,146 @@ github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa h1:XKAhUk/dtp+CV github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/fatih/color v1.3.0 h1:YehCCcyeQ6Km0D6+IapqPinWBK6y+0eB5umvZXK9WPs= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc h1:jtW8jbpkO4YirRSyepBOH8E+2HEw6/hKkBvFPwhUN8c= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c h1:zqAKixg3cTcIasAMJV+EcfVbWwLpOZ7LeoWJvcuD/5Q= github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989 h1:giknQ4mEuDFmmHSrGcbargOuLHQGtywqo4mheITex54= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277 h1:E0whKxgp2ojts0FDgUA8dl62bmH0LxKanMoBr6MDTDM= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad h1:eMxs9EL0PvIGS9TTtxg4R+JxuPGav82J8rA+GFnY7po= github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3 h1:DqD8eigqlUm0+znmx7zhL0xvTW3+e1jCekJMfBUADWI= github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM= +github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w= +github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g= +github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE= +github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= +github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -124,17 +246,63 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= @@ -148,10 +316,14 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0dQbyuPWGNQJZGPwPu/QS9GlpnA= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -159,23 +331,52 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN8iVhff6M38Mfu73FQiJve/GEXYJBjE= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9 h1:5Cp3cVwpQP4aCQ6jx6dNLP3IarbYiuStmIzYu+BjQwY= github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= @@ -183,54 +384,185 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk= github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 h1:hhsSf/5z74Ck/DJYc+R8zpq8KGm7uJvpdLRQED/IedA= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/statediff/api.go b/statediff/api.go index b395557bb52d..1fffd90a13f7 100644 --- a/statediff/api.go +++ b/statediff/api.go @@ -131,3 +131,8 @@ func (api *PublicStateDiffAPI) StreamCodeAndCodeHash(ctx context.Context, blockN return rpcSub, nil } + +// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight +func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params Params) error { + return api.sds.WriteStateDiffAt(blockNumber, params) +} diff --git a/statediff/builder.go b/statediff/builder.go index c84678d51a69..d9ccc9f3bbf0 100644 --- a/statediff/builder.go +++ b/statediff/builder.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" "github.com/ethereum/go-ethereum/trie" ) @@ -43,12 +44,27 @@ var ( type Builder interface { BuildStateDiffObject(args Args, params Params) (StateObject, error) BuildStateTrieObject(current *types.Block) (StateObject, error) + WriteStateDiffObject(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) } type builder struct { stateCache state.Database } +// convenience +func stateNodeAppender(nodes *[]sdtypes.StateNode) sdtypes.StateNodeSink { + return func(node sdtypes.StateNode) error { + *nodes = append(*nodes, node) + return nil + } +} +func storageNodeAppender(nodes *[]sdtypes.StorageNode) sdtypes.StorageNodeSink { + return func(node sdtypes.StorageNode) error { + *nodes = append(*nodes, node) + return nil + } +} + // NewBuilder is used to create a statediff builder func NewBuilder(stateCache state.Database) Builder { return &builder{ @@ -75,8 +91,8 @@ func (sdb *builder) BuildStateTrieObject(current *types.Block) (StateObject, err }, nil } -func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAndCodeHash, error) { - stateNodes := make([]StateNode, 0) +func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, []CodeAndCodeHash, error) { + stateNodes := make([]sdtypes.StateNode, 0) codeAndCodeHashes := make([]CodeAndCodeHash, 0) for it.Next(true) { // skip value nodes @@ -101,7 +117,7 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAnd return nil, nil, err } switch ty { - case Leaf: + case sdtypes.Leaf: var account state.Account if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) @@ -110,14 +126,15 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAnd valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] - node := StateNode{ + node := sdtypes.StateNode{ NodeType: ty, Path: nodePath, LeafKey: leafKey, NodeValue: node, } if !bytes.Equal(account.CodeHash, nullCodeHash) { - storageNodes, err := sdb.buildStorageNodesEventual(account.Root, nil, true) + var storageNodes []sdtypes.StorageNode + err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes)) if err != nil { return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err) } @@ -134,8 +151,8 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAnd }) } stateNodes = append(stateNodes, node) - case Extension, Branch: - stateNodes = append(stateNodes, StateNode{ + case sdtypes.Extension, sdtypes.Branch: + stateNodes = append(stateNodes, sdtypes.StateNode{ NodeType: ty, Path: nodePath, NodeValue: node, @@ -149,36 +166,59 @@ func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]StateNode, []CodeAnd // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters func (sdb *builder) BuildStateDiffObject(args Args, params Params) (StateObject, error) { - if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 { // if we are watching only specific accounts then we are only diffing leaf nodes - return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params) + var stateNodes []sdtypes.StateNode + codeAndCodeHashes, err := sdb.WriteStateDiffObject(StateRoots{ + OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot, + }, params, stateNodeAppender(&stateNodes)) + if err != nil { + return StateObject{}, err + } + return StateObject{ + BlockHash: args.BlockHash, + BlockNumber: args.BlockNumber, + Nodes: stateNodes, + CodeAndCodeHashes: codeAndCodeHashes, + }, nil +} + +// Writes a statediff object to output callback +func (sdb *builder) WriteStateDiffObject(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { + if !params.IntermediateStateNodes || len(params.WatchedAddresses) > 0 { + // if we are watching only specific accounts then we are only diffing leaf nodes + return sdb.buildStateDiffWithoutIntermediateStateNodes(args, params, output) + } else { + return sdb.buildStateDiffWithIntermediateStateNodes(args, params, output) } - return sdb.buildStateDiffWithIntermediateStateNodes(args, params) } -func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args Args, params Params) (StateObject, error) { +func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { // Load tries for old and new states oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) if err != nil { - return StateObject{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) + return nil, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) if err != nil { - return StateObject{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) + return nil, fmt.Errorf("error creating trie for newStateRoot: %v", err) } // collect a slice of all the intermediate nodes that were touched and exist at B // a map of their leafkey to all the accounts that were touched and exist at B // and a slice of all the paths for the nodes in both of the above sets - createdOrUpdatedIntermediateNodes, diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStateWithIntermediateNodes( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + output) if err != nil { - return StateObject{}, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + return nil, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) } // collect a slice of all the nodes that existed at a path in A that doesn't exist in B // a map of their leafkey to all the accounts that were touched and exist at A - emptiedPaths, diffAccountsAtA, err := sdb.deletedOrUpdatedState(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB) + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, output) if err != nil { - return StateObject{}, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + return nil, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } // collect and sort the leafkey keys for both account mappings into a slice @@ -192,48 +232,47 @@ func (sdb *builder) buildStateDiffWithIntermediateStateNodes(args Args, params P updatedKeys := findIntersection(createKeys, deleteKeys) // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two - updatedAccounts, err := sdb.buildAccountUpdates(diffAccountsAtB, diffAccountsAtA, updatedKeys, params.WatchedStorageSlots, params.IntermediateStorageNodes) + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return StateObject{}, fmt.Errorf("error building diff for updated accounts: %v", err) + return nil, fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - createdAccounts, codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes) + codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return StateObject{}, fmt.Errorf("error building diff for created accounts: %v", err) + return nil, fmt.Errorf("error building diff for created accounts: %v", err) } - - // assemble all of the nodes into the statediff object, including the intermediate nodes - return StateObject{ - BlockNumber: args.BlockNumber, - BlockHash: args.BlockHash, - Nodes: append(append(append(updatedAccounts, createdAccounts...), createdOrUpdatedIntermediateNodes...), emptiedPaths...), - CodeAndCodeHashes: codeAndCodeHashes, - }, nil + return codeAndCodeHashes, nil } -func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args Args, params Params) (StateObject, error) { +func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args StateRoots, params Params, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { // Load tries for old (A) and new (B) states oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot) if err != nil { - return StateObject{}, fmt.Errorf("error creating trie for oldStateRoot: %v", err) + return nil, fmt.Errorf("error creating trie for oldStateRoot: %v", err) } newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot) if err != nil { - return StateObject{}, fmt.Errorf("error creating trie for newStateRoot: %v", err) + return nil, fmt.Errorf("error creating trie for newStateRoot: %v", err) } // collect a map of their leafkey to all the accounts that were touched and exist at B // and a slice of all the paths for the nodes in both of the above sets - diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), params.WatchedAddresses) + diffAccountsAtB, diffPathsAtB, err := sdb.createdAndUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + params.WatchedAddresses) if err != nil { - return StateObject{}, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) + return nil, fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err) } // collect a slice of all the nodes that existed at a path in A that doesn't exist in B // a map of their leafkey to all the accounts that were touched and exist at A - emptiedPaths, diffAccountsAtA, err := sdb.deletedOrUpdatedState(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB) + diffAccountsAtA, err := sdb.deletedOrUpdatedState( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, output) if err != nil { - return StateObject{}, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) + return nil, fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err) } // collect and sort the leafkeys for both account mappings into a slice @@ -247,23 +286,18 @@ func (sdb *builder) buildStateDiffWithoutIntermediateStateNodes(args Args, param updatedKeys := findIntersection(createKeys, deleteKeys) // build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two - updatedAccounts, err := sdb.buildAccountUpdates(diffAccountsAtB, diffAccountsAtA, updatedKeys, params.WatchedStorageSlots, params.IntermediateStorageNodes) + err = sdb.buildAccountUpdates( + diffAccountsAtB, diffAccountsAtA, updatedKeys, + params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return StateObject{}, fmt.Errorf("error building diff for updated accounts: %v", err) + return nil, fmt.Errorf("error building diff for updated accounts: %v", err) } // build the diff nodes for created accounts - createdAccounts, codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes) + codeAndCodeHashes, err := sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output) if err != nil { - return StateObject{}, fmt.Errorf("error building diff for created accounts: %v", err) + return nil, fmt.Errorf("error building diff for created accounts: %v", err) } - - // assemble all of the nodes into the statediff object - return StateObject{ - BlockNumber: args.BlockNumber, - BlockHash: args.BlockHash, - Nodes: append(append(updatedAccounts, createdAccounts...), emptiedPaths...), - CodeAndCodeHashes: codeAndCodeHashes, - }, nil + return codeAndCodeHashes, nil } // createdAndUpdatedState returns @@ -295,7 +329,7 @@ func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddres if err != nil { return nil, nil, err } - if ty == Leaf { + if ty == sdtypes.Leaf { // created vs updated is important for leaf nodes since we need to diff their storage // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey var account state.Account @@ -326,8 +360,7 @@ func (sdb *builder) createdAndUpdatedState(a, b trie.NodeIterator, watchedAddres // a slice of all the intermediate nodes that exist in a different state at B than A // a mapping of their leafkeys to all the accounts that exist in a different state at B than A // and a slice of the paths for all of the nodes included in both -func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator) ([]StateNode, AccountMap, map[string]bool, error) { - createdOrUpdatedIntermediateNodes := make([]StateNode, 0) +func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIterator, output sdtypes.StateNodeSink) (AccountMap, map[string]bool, error) { diffPathsAtB := make(map[string]bool) diffAcountsAtB := make(AccountMap) it, _ := trie.NewDifferenceIterator(a, b) @@ -343,23 +376,23 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt copy(nodePath, it.Path()) node, err := sdb.stateCache.TrieDB().Node(it.Hash()) if err != nil { - return nil, nil, nil, err + return nil, nil, err } var nodeElements []interface{} if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, nil, nil, err + return nil, nil, err } ty, err := CheckKeyType(nodeElements) if err != nil { - return nil, nil, nil, err + return nil, nil, err } switch ty { - case Leaf: + case sdtypes.Leaf: // created vs updated is important for leaf nodes since we need to diff their storage - // so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey + // so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey var account state.Account if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { - return nil, nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) + return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) } partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) @@ -372,27 +405,28 @@ func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(a, b trie.NodeIt LeafKey: leafKey, Account: &account, } - case Extension, Branch: + case sdtypes.Extension, sdtypes.Branch: // create a diff for any intermediate node that has changed at b // created vs updated makes no difference for intermediate nodes since we do not need to diff storage - createdOrUpdatedIntermediateNodes = append(createdOrUpdatedIntermediateNodes, StateNode{ + if err := output(sdtypes.StateNode{ NodeType: ty, Path: nodePath, NodeValue: node, - }) + }); err != nil { + return nil, nil, err + } default: - return nil, nil, nil, fmt.Errorf("unexpected node type %s", ty) + return nil, nil, fmt.Errorf("unexpected node type %s", ty) } // add both intermediate and leaf node paths to the list of diffPathsAtB diffPathsAtB[common.Bytes2Hex(nodePath)] = true } - return createdOrUpdatedIntermediateNodes, diffAcountsAtB, diffPathsAtB, it.Error() + return diffAcountsAtB, diffPathsAtB, it.Error() } // deletedOrUpdatedState returns a slice of all the pathes that are emptied at B // and a mapping of their leafkeys to all the accounts that exist in a different state at A than B -func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool) ([]StateNode, AccountMap, error) { - emptiedPaths := make([]StateNode, 0) +func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB map[string]bool, output sdtypes.StateNodeSink) (AccountMap, error) { diffAccountAtA := make(AccountMap) it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { @@ -409,30 +443,32 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m // that means the node at this path was deleted (or moved) in B // emit an empty "removed" diff to signify as such if _, ok := diffPathsAtB[common.Bytes2Hex(nodePath)]; !ok { - emptiedPaths = append(emptiedPaths, StateNode{ + if err := output(sdtypes.StateNode{ Path: nodePath, NodeValue: []byte{}, - NodeType: Removed, - }) + NodeType: sdtypes.Removed, + }); err != nil { + return nil, err + } } node, err := sdb.stateCache.TrieDB().Node(it.Hash()) if err != nil { - return nil, nil, err + return nil, err } var nodeElements []interface{} if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, nil, err + return nil, err } ty, err := CheckKeyType(nodeElements) if err != nil { - return nil, nil, err + return nil, err } switch ty { - case Leaf: + case sdtypes.Leaf: // map all different accounts at A to their leafkey var account state.Account if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil { - return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) + return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", nodePath, err) } partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) @@ -445,55 +481,58 @@ func (sdb *builder) deletedOrUpdatedState(a, b trie.NodeIterator, diffPathsAtB m LeafKey: leafKey, Account: &account, } - case Extension, Branch: + case sdtypes.Extension, sdtypes.Branch: // fall through, we did everything we need to do with these node types default: - return nil, nil, fmt.Errorf("unexpected node type %s", ty) + return nil, fmt.Errorf("unexpected node type %s", ty) } } - return emptiedPaths, diffAccountAtA, it.Error() + return diffAccountAtA, it.Error() } // buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys // to generate the statediff node objects for all of the accounts that existed at both A and B but in different states // needs to be called before building account creations and deletions as this mutates // those account maps to remove the accounts which were updated -func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, watchedStorageKeys []common.Hash, intermediateStorageNodes bool) ([]StateNode, error) { - updatedAccounts := make([]StateNode, 0, len(updatedKeys)) +func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, + watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error { var err error for _, key := range updatedKeys { createdAcc := creations[key] deletedAcc := deletions[key] - var storageDiffs []StorageNode + var storageDiffs []sdtypes.StorageNode if deletedAcc.Account != nil && createdAcc.Account != nil { oldSR := deletedAcc.Account.Root newSR := createdAcc.Account.Root - storageDiffs, err = sdb.buildStorageNodesIncremental(oldSR, newSR, watchedStorageKeys, intermediateStorageNodes) + err = sdb.buildStorageNodesIncremental( + oldSR, newSR, watchedStorageKeys, intermediateStorageNodes, + storageNodeAppender(&storageDiffs)) if err != nil { - return nil, fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) + return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err) } } - updatedAccounts = append(updatedAccounts, StateNode{ + if err = output(sdtypes.StateNode{ NodeType: createdAcc.NodeType, Path: createdAcc.Path, NodeValue: createdAcc.NodeValue, LeafKey: createdAcc.LeafKey, StorageNodes: storageDiffs, - }) + }); err != nil { + return err + } delete(creations, key) delete(deletions, key) } - return updatedAccounts, nil + return nil } // buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A // it also returns the code and codehash for created contract accounts -func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool) ([]StateNode, []CodeAndCodeHash, error) { - accountDiffs := make([]StateNode, 0, len(accounts)) +func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) ([]CodeAndCodeHash, error) { codeAndCodeHashes := make([]CodeAndCodeHash, 0) for _, val := range accounts { - diff := StateNode{ + diff := sdtypes.StateNode{ NodeType: val.NodeType, Path: val.Path, LeafKey: val.LeafKey, @@ -501,49 +540,55 @@ func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKey } if !bytes.Equal(val.Account.CodeHash, nullCodeHash) { // For contract creations, any storage node contained is a diff - storageDiffs, err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes) + var storageDiffs []sdtypes.StorageNode + err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs)) if err != nil { - return nil, nil, fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) + return nil, fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err) } diff.StorageNodes = storageDiffs // emit codehash => code mappings for cod codeHash := common.BytesToHash(val.Account.CodeHash) code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash) if err != nil { - return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) + return nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err) } codeAndCodeHashes = append(codeAndCodeHashes, CodeAndCodeHash{ Hash: codeHash, Code: code, }) } - accountDiffs = append(accountDiffs, diff) + if err := output(diff); err != nil { + return nil, err + } } - return accountDiffs, codeAndCodeHashes, nil + return codeAndCodeHashes, nil } // buildStorageNodesEventual builds the storage diff node objects for a created account // i.e. it returns all the storage nodes at this state, since there is no previous state -func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) { +func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) { - return nil, nil + return nil } log.Debug("Storage Root For Eventual Diff", "root", sr.Hex()) sTrie, err := sdb.stateCache.OpenTrie(sr) if err != nil { log.Info("error in build storage diff eventual", "error", err) - return nil, err + return err } it := sTrie.NodeIterator(make([]byte, 0)) - return sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes) + err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output) + if err != nil { + return err + } + return nil } // buildStorageNodesFromTrie returns all the storage diff node objects in the provided node interator // if any storage keys are provided it will only return those leaf nodes // including intermediate nodes can be turned on or off -func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) { - storageDiffs := make([]StorageNode, 0) +func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { for it.Next(true) { // skip value nodes if it.Leaf() { @@ -556,73 +601,79 @@ func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStora copy(nodePath, it.Path()) node, err := sdb.stateCache.TrieDB().Node(it.Hash()) if err != nil { - return nil, err + return err } var nodeElements []interface{} if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, err + return err } ty, err := CheckKeyType(nodeElements) if err != nil { - return nil, err + return err } switch ty { - case Leaf: + case sdtypes.Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedStorageKeys, leafKey) { - storageDiffs = append(storageDiffs, StorageNode{ + if err := output(sdtypes.StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, LeafKey: leafKey, - }) + }); err != nil { + return err + } } - case Extension, Branch: + case sdtypes.Extension, sdtypes.Branch: if intermediateNodes { - storageDiffs = append(storageDiffs, StorageNode{ + if err := output(sdtypes.StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, - }) + }); err != nil { + return err + } } default: - return nil, fmt.Errorf("unexpected node type %s", ty) + return fmt.Errorf("unexpected node type %s", ty) } } - return storageDiffs, it.Error() + return it.Error() } // buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A -func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) { +func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) { - return nil, nil + return nil } log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex()) oldTrie, err := sdb.stateCache.OpenTrie(oldSR) if err != nil { - return nil, err + return err } newTrie, err := sdb.stateCache.OpenTrie(newSR) if err != nil { - return nil, err + return err } - createdOrUpdatedStorage, diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), watchedStorageKeys, intermediateNodes) + diffPathsAtB, err := sdb.createdAndUpdatedStorage( + oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + watchedStorageKeys, intermediateNodes, output) if err != nil { - return nil, err + return err } - deletedStorage, err := sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, watchedStorageKeys, intermediateNodes) + err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), + diffPathsAtB, watchedStorageKeys, intermediateNodes, output) if err != nil { - return nil, err + return err } - return append(createdOrUpdatedStorage, deletedStorage...), nil + return nil } -func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool) ([]StorageNode, map[string]bool, error) { - createdOrUpdatedStorage := make([]StorageNode, 0) +func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, error) { diffPathsAtB := make(map[string]bool) it, _ := trie.NewDifferenceIterator(a, b) for it.Next(true) { @@ -637,48 +688,51 @@ func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys copy(nodePath, it.Path()) node, err := sdb.stateCache.TrieDB().Node(it.Hash()) if err != nil { - return nil, nil, err + return nil, err } var nodeElements []interface{} if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, nil, err + return nil, err } ty, err := CheckKeyType(nodeElements) if err != nil { - return nil, nil, err + return nil, err } switch ty { - case Leaf: + case sdtypes.Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedKeys, leafKey) { - createdOrUpdatedStorage = append(createdOrUpdatedStorage, StorageNode{ + if err := output(sdtypes.StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, LeafKey: leafKey, - }) + }); err != nil { + return nil, err + } } - case Extension, Branch: + case sdtypes.Extension, sdtypes.Branch: if intermediateNodes { - createdOrUpdatedStorage = append(createdOrUpdatedStorage, StorageNode{ + if err := output(sdtypes.StorageNode{ NodeType: ty, Path: nodePath, NodeValue: node, - }) + }); err != nil { + return nil, err + } } default: - return nil, nil, fmt.Errorf("unexpected node type %s", ty) + return nil, fmt.Errorf("unexpected node type %s", ty) } diffPathsAtB[common.Bytes2Hex(nodePath)] = true } - return createdOrUpdatedStorage, diffPathsAtB, it.Error() + return diffPathsAtB, it.Error() } -func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool) ([]StorageNode, error) { - deletedStorage := make([]StorageNode, 0) +func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error { it, _ := trie.NewDifferenceIterator(b, a) for it.Next(true) { // skip value nodes @@ -698,42 +752,46 @@ func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB } node, err := sdb.stateCache.TrieDB().Node(it.Hash()) if err != nil { - return nil, err + return err } var nodeElements []interface{} if err := rlp.DecodeBytes(node, &nodeElements); err != nil { - return nil, err + return err } ty, err := CheckKeyType(nodeElements) if err != nil { - return nil, err + return err } switch ty { - case Leaf: + case sdtypes.Leaf: partialPath := trie.CompactToHex(nodeElements[0].([]byte)) valueNodePath := append(nodePath, partialPath...) encodedPath := trie.HexToCompact(valueNodePath) leafKey := encodedPath[1:] if isWatchedStorageKey(watchedKeys, leafKey) { - deletedStorage = append(deletedStorage, StorageNode{ - NodeType: Removed, + if err := output(sdtypes.StorageNode{ + NodeType: sdtypes.Removed, Path: nodePath, NodeValue: []byte{}, - }) + }); err != nil { + return err + } } - case Extension, Branch: + case sdtypes.Extension, sdtypes.Branch: if intermediateNodes { - deletedStorage = append(deletedStorage, StorageNode{ - NodeType: Removed, + if err := output(sdtypes.StorageNode{ + NodeType: sdtypes.Removed, Path: nodePath, NodeValue: []byte{}, - }) + }); err != nil { + return err + } } default: - return nil, fmt.Errorf("unexpected node type %s", ty) + return fmt.Errorf("unexpected node type %s", ty) } } - return deletedStorage, it.Error() + return it.Error() } // isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch diff --git a/statediff/builder_test.go b/statediff/builder_test.go index a2f1090afd7b..2e6c149480d8 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -29,13 +29,14 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) // TODO: add test that filters on address var ( contractLeafKey []byte - emptyDiffs = make([]statediff.StateNode, 0) - emptyStorage = make([]statediff.StorageNode, 0) + emptyDiffs = make([]sdtypes.StateNode, 0) + emptyStorage = make([]sdtypes.StorageNode, 0) block0, block1, block2, block3, block4, block5, block6 *types.Block builder statediff.Builder miningReward = int64(2000000000000000000) @@ -505,10 +506,10 @@ func TestBuilder(t *testing.T) { &statediff.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock0LeafNode, StorageNodes: emptyStorage, @@ -528,24 +529,24 @@ func TestBuilder(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock1LeafNode, StorageNodes: emptyStorage, @@ -567,43 +568,43 @@ func TestBuilder(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock2LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot0StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -611,7 +612,7 @@ func TestBuilder(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock2LeafNode, StorageNodes: emptyStorage, @@ -638,23 +639,23 @@ func TestBuilder(t *testing.T) { &statediff.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock3LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock3LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot3StorageKey.Bytes(), NodeValue: slot3StorageLeafNode, }, @@ -662,7 +663,7 @@ func TestBuilder(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock3LeafNode, StorageNodes: emptyStorage, @@ -740,10 +741,10 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { &statediff.StateObject{ BlockNumber: block0.Number(), BlockHash: block0.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock0LeafNode, StorageNodes: emptyStorage, @@ -763,30 +764,30 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block1BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock1LeafNode, StorageNodes: emptyStorage, @@ -808,54 +809,54 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block2BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock2LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block2StorageBranchRootNode, }, { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot0StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -863,7 +864,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock2LeafNode, StorageNodes: emptyStorage, @@ -890,34 +891,34 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { &statediff.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block3BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock3LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock3LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block3StorageBranchRootNode, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot3StorageKey.Bytes(), NodeValue: slot3StorageLeafNode, }, @@ -925,7 +926,7 @@ func TestBuilderWithIntermediateNodes(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock3LeafNode, StorageNodes: emptyStorage, @@ -1030,10 +1031,10 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock1LeafNode, StorageNodes: emptyStorage, @@ -1054,22 +1055,22 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock2LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot0StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -1077,7 +1078,7 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, @@ -1104,16 +1105,16 @@ func TestBuilderWithWatchedAddressList(t *testing.T) { &statediff.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock3LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot3StorageKey.Bytes(), NodeValue: slot3StorageLeafNode, }, @@ -1206,10 +1207,10 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock1LeafNode, StorageNodes: emptyStorage, @@ -1230,16 +1231,16 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock2LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -1247,7 +1248,7 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, @@ -1274,10 +1275,10 @@ func TestBuilderWithWatchedAddressAndStorageKeyList(t *testing.T) { &statediff.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock3LeafNode, StorageNodes: emptyStorage, @@ -1340,52 +1341,52 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { &statediff.StateObject{ BlockNumber: block4.Number(), BlockHash: block4.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block4BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock4LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock4LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block4StorageBranchRootNode, }, { Path: []byte{'\x04'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot2StorageKey.Bytes(), NodeValue: slot2StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock4LeafNode, StorageNodes: emptyStorage, @@ -1404,47 +1405,47 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { &statediff.StateObject{ BlockNumber: block5.Number(), BlockHash: block5.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block5BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock5LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock5LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, NodeValue: slot0StorageLeafRootNode, LeafKey: slot0StorageKey.Bytes(), }, { Path: []byte{'\x02'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x04'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock5LeafNode, StorageNodes: emptyStorage, @@ -1463,28 +1464,28 @@ func TestBuilderWithRemovedAccountAndStorage(t *testing.T) { &statediff.StateObject{ BlockNumber: block6.Number(), BlockHash: block6.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block6BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock6LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock6LeafNode, StorageNodes: emptyStorage, @@ -1547,41 +1548,41 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. &statediff.StateObject{ BlockNumber: block4.Number(), BlockHash: block4.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock4LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock4LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x04'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot2StorageKey.Bytes(), NodeValue: slot2StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock4LeafNode, StorageNodes: emptyStorage, @@ -1600,41 +1601,41 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. &statediff.StateObject{ BlockNumber: block5.Number(), BlockHash: block5.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock5LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock5LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, NodeValue: slot0StorageLeafRootNode, LeafKey: slot0StorageKey.Bytes(), }, { Path: []byte{'\x02'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x04'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock5LeafNode, StorageNodes: emptyStorage, @@ -1653,22 +1654,22 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. &statediff.StateObject{ BlockNumber: block6.Number(), BlockHash: block6.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x06'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock6LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock6LeafNode, StorageNodes: emptyStorage, @@ -1810,40 +1811,40 @@ func TestBuilderWithMovedAccount(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block01BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock01LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x01'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock01LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block01StorageBranchRootNode, }, { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot00StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -1869,22 +1870,22 @@ func TestBuilderWithMovedAccount(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock02LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x01'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x00'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, @@ -1943,29 +1944,29 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock01LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x01'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock01LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot00StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -1991,22 +1992,22 @@ func TestBuilderWithMovedAccountOnlyLeafs(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock02LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x01'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, { Path: []byte{'\x00'}, - NodeType: statediff.Removed, + NodeType: sdtypes.Removed, NodeValue: []byte{}, }, }, @@ -2056,30 +2057,30 @@ func TestBuildStateTrie(t *testing.T) { &statediff.StateObject{ BlockNumber: block1.Number(), BlockHash: block1.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block1BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock1LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock1LeafNode, StorageNodes: emptyStorage, @@ -2093,54 +2094,54 @@ func TestBuildStateTrie(t *testing.T) { &statediff.StateObject{ BlockNumber: block2.Number(), BlockHash: block2.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block2BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock2LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block2StorageBranchRootNode, }, { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot0StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, @@ -2148,7 +2149,7 @@ func TestBuildStateTrie(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock2LeafNode, StorageNodes: emptyStorage, @@ -2168,60 +2169,60 @@ func TestBuildStateTrie(t *testing.T) { &statediff.StateObject{ BlockNumber: block3.Number(), BlockHash: block3.Hash(), - Nodes: []statediff.StateNode{ + Nodes: []sdtypes.StateNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block3BranchRootNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x00'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.BankLeafKey, NodeValue: bankAccountAtBlock3LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x05'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: minerLeafKey, NodeValue: minerAccountAtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x0e'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account1LeafKey, NodeValue: account1AtBlock2LeafNode, StorageNodes: emptyStorage, }, { Path: []byte{'\x06'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: contractLeafKey, NodeValue: contractAccountAtBlock3LeafNode, - StorageNodes: []statediff.StorageNode{ + StorageNodes: []sdtypes.StorageNode{ { Path: []byte{}, - NodeType: statediff.Branch, + NodeType: sdtypes.Branch, NodeValue: block3StorageBranchRootNode, }, { Path: []byte{'\x02'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot0StorageKey.Bytes(), NodeValue: slot0StorageLeafNode, }, { Path: []byte{'\x0b'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot1StorageKey.Bytes(), NodeValue: slot1StorageLeafNode, }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: slot3StorageKey.Bytes(), NodeValue: slot3StorageLeafNode, }, @@ -2229,7 +2230,7 @@ func TestBuildStateTrie(t *testing.T) { }, { Path: []byte{'\x0c'}, - NodeType: statediff.Leaf, + NodeType: sdtypes.Leaf, LeafKey: testhelpers.Account2LeafKey, NodeValue: account2AtBlock3LeafNode, StorageNodes: emptyStorage, diff --git a/statediff/helpers.go b/statediff/helpers.go index 05db7b00a90b..51ac5c1be80f 100644 --- a/statediff/helpers.go +++ b/statediff/helpers.go @@ -23,6 +23,8 @@ import ( "fmt" "sort" "strings" + + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) func sortKeys(data AccountMap) []string { @@ -74,23 +76,23 @@ func findIntersection(a, b []string) []string { } // CheckKeyType checks what type of key we have -func CheckKeyType(elements []interface{}) (NodeType, error) { +func CheckKeyType(elements []interface{}) (sdtypes.NodeType, error) { if len(elements) > 2 { - return Branch, nil + return sdtypes.Branch, nil } if len(elements) < 2 { - return Unknown, fmt.Errorf("node cannot be less than two elements in length") + return sdtypes.Unknown, fmt.Errorf("node cannot be less than two elements in length") } switch elements[0].([]byte)[0] / 16 { case '\x00': - return Extension, nil + return sdtypes.Extension, nil case '\x01': - return Extension, nil + return sdtypes.Extension, nil case '\x02': - return Leaf, nil + return sdtypes.Leaf, nil case '\x03': - return Leaf, nil + return sdtypes.Leaf, nil default: - return Unknown, fmt.Errorf("unknown hex prefix") + return sdtypes.Unknown, fmt.Errorf("unknown hex prefix") } } diff --git a/statediff/indexer/helpers.go b/statediff/indexer/helpers.go new file mode 100644 index 000000000000..4865cafa1c88 --- /dev/null +++ b/statediff/indexer/helpers.go @@ -0,0 +1,55 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/statediff/types" +) + +func ResolveFromNodeType(nodeType types.NodeType) int { + switch nodeType { + case types.Branch: + return 0 + case types.Extension: + return 1 + case types.Leaf: + return 2 + case types.Removed: + return 3 + default: + return -1 + } +} + +// ChainConfig returns the appropriate ethereum chain config for the provided chain id +func ChainConfig(chainID uint64) (*params.ChainConfig, error) { + switch chainID { + case 1: + return params.MainnetChainConfig, nil + case 3: + return params.TestnetChainConfig, nil // Ropsten + case 4: + return params.RinkebyChainConfig, nil + case 5: + return params.GoerliChainConfig, nil + default: + return nil, fmt.Errorf("chain config for chainid %d not available", chainID) + } +} diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go new file mode 100644 index 000000000000..117a0bf17ae0 --- /dev/null +++ b/statediff/indexer/indexer.go @@ -0,0 +1,366 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + + node "github.com/ipfs/go-ipld-format" + "github.com/jmoiron/sqlx" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// Indexer interface to allow substitution of mocks for testing +type Indexer interface { + PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) + PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error +} + +// StateDiffIndexer satisfies the Indexer interface for ethereum statediff objects +type StateDiffIndexer struct { + chainConfig *params.ChainConfig + indexer *PostgresCIDWriter +} + +// NewStateDiffIndexer creates a pointer to a new PayloadConverter which satisfies the PayloadConverter interface +func NewStateDiffIndexer(chainConfig *params.ChainConfig, db *postgres.DB) *StateDiffIndexer { + return &StateDiffIndexer{ + chainConfig: chainConfig, + indexer: NewPostgresCIDWriter(db), + } +} + +type BlockTx struct { + dbtx *sqlx.Tx + BlockNumber uint64 + headerID int64 + err error + Close func() error +} + +// Pushes and indexes block data in database, excluding state & storage nodes (header, uncles, transactions & receipts) +// Returns an initiated DB transaction which must be Closed via defer to commit or rollback +func (sdt *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (*BlockTx, error) { + start, t := time.Now(), time.Now() + blockHash := block.Hash() + blockHashStr := blockHash.String() + height := block.NumberU64() + traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr) + transactions := block.Transactions() + // Derive any missing fields + if err := receipts.DeriveFields(sdt.chainConfig, blockHash, height, transactions); err != nil { + return nil, err + } + // Generate the block iplds + headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(block, receipts) + if err != nil { + return nil, err + } + if len(txNodes) != len(txTrieNodes) && len(rctNodes) != len(rctTrieNodes) && len(txNodes) != len(rctNodes) { + return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes)) + } + // Calculate reward + reward := CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + traceMsg += fmt.Sprintf("payload decoding duration: %s\r\n", time.Now().Sub(t).String()) + t = time.Now() + // Begin new db tx for everything + tx, err := sdt.indexer.db.Beginx() + if err != nil { + return nil, err + } + blocktx := BlockTx{ + dbtx: tx, + // handle transaction commit or rollback for any return case + Close: func() error { + var err error + if p := recover(); p != nil { + shared.Rollback(tx) + panic(p) + } else { + err = tx.Commit() + traceMsg += fmt.Sprintf("postgres transaction commit duration: %s\r\n", time.Now().Sub(t).String()) + } + traceMsg += fmt.Sprintf(" TOTAL PROCESSING DURATION: %s\r\n", time.Now().Sub(start).String()) + log.Info(traceMsg) + return err + }, + } + traceMsg += fmt.Sprintf("time spent waiting for free postgres tx: %s:\r\n", time.Now().Sub(t).String()) + t = time.Now() + + // Publish and index header, collect headerID + headerID, err := sdt.processHeader(tx, block.Header(), headerNode, reward, totalDifficulty) + if err != nil { + return nil, err + } + traceMsg += fmt.Sprintf("header processing duration: %s\r\n", time.Now().Sub(t).String()) + t = time.Now() + // Publish and index uncles + if err := sdt.processUncles(tx, headerID, height, uncleNodes); err != nil { + return nil, err + } + traceMsg += fmt.Sprintf("uncle processing duration: %s\r\n", time.Now().Sub(t).String()) + t = time.Now() + // Publish and index receipts and txs + if err := sdt.processReceiptsAndTxs(tx, processArgs{ + headerID: headerID, + blockNumber: block.Number(), + receipts: receipts, + txs: transactions, + rctNodes: rctNodes, + rctTrieNodes: rctTrieNodes, + txNodes: txNodes, + txTrieNodes: txTrieNodes, + }); err != nil { + return nil, err + } + traceMsg += fmt.Sprintf("tx and receipt processing duration: %s\r\n", time.Now().Sub(t).String()) + // t = time.Now() + blocktx.BlockNumber = height + blocktx.headerID = headerID + return &blocktx, err // return error explicity so that the defer() assigns to it +} + +// processHeader publishes and indexes a header IPLD in Postgres +// it returns the headerID +func (sdt *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, headerNode node.Node, reward, td *big.Int) (int64, error) { + // publish header + if err := shared.PublishIPLD(tx, headerNode); err != nil { + return 0, err + } + // index header + return sdt.indexer.indexHeaderCID(tx, models.HeaderModel{ + CID: headerNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), + ParentHash: header.ParentHash.String(), + BlockNumber: header.Number.String(), + BlockHash: header.Hash().String(), + TotalDifficulty: td.String(), + Reward: reward.String(), + Bloom: header.Bloom.Bytes(), + StateRoot: header.Root.String(), + RctRoot: header.ReceiptHash.String(), + TxRoot: header.TxHash.String(), + UncleRoot: header.UncleHash.String(), + Timestamp: header.Time, + }) +} + +func (sdt *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNumber uint64, uncleNodes []*ipld.EthHeader) error { + // publish and index uncles + for _, uncleNode := range uncleNodes { + if err := shared.PublishIPLD(tx, uncleNode); err != nil { + return err + } + uncleReward := CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) + uncle := models.UncleModel{ + CID: uncleNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), + ParentHash: uncleNode.ParentHash.String(), + BlockHash: uncleNode.Hash().String(), + Reward: uncleReward.String(), + } + if err := sdt.indexer.indexUncleCID(tx, uncle, headerID); err != nil { + return err + } + } + return nil +} + +// processArgs bundles arugments to processReceiptsAndTxs +type processArgs struct { + headerID int64 + blockNumber *big.Int + receipts types.Receipts + txs types.Transactions + rctNodes []*ipld.EthReceipt + rctTrieNodes []*ipld.EthRctTrie + txNodes []*ipld.EthTx + txTrieNodes []*ipld.EthTxTrie +} + +// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres +func (sdt *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs) error { + // Process receipts and txs + signer := types.MakeSigner(sdt.chainConfig, args.blockNumber) + for i, receipt := range args.receipts { + // tx that corresponds with this receipt + trx := args.txs[i] + from, err := types.Sender(signer, trx) + if err != nil { + return err + } + + // Publishing + // publish trie nodes, these aren't indexed directly + if err := shared.PublishIPLD(tx, args.txTrieNodes[i]); err != nil { + return err + } + if err := shared.PublishIPLD(tx, args.rctTrieNodes[i]); err != nil { + return err + } + // publish the txs and receipts + txNode, rctNode := args.txNodes[i], args.rctNodes[i] + if err := shared.PublishIPLD(tx, txNode); err != nil { + return err + } + if err := shared.PublishIPLD(tx, rctNode); err != nil { + return err + } + + // Indexing + // extract topic and contract data from the receipt for indexing + topicSets := make([][]string, 4) + mappedContracts := make(map[string]bool) // use map to avoid duplicate addresses + for _, log := range receipt.Logs { + for i, topic := range log.Topics { + topicSets[i] = append(topicSets[i], topic.Hex()) + } + mappedContracts[log.Address.String()] = true + } + // these are the contracts seen in the logs + logContracts := make([]string, 0, len(mappedContracts)) + for addr := range mappedContracts { + logContracts = append(logContracts, addr) + } + // this is the contract address if this receipt is for a contract creation tx + contract := shared.HandleZeroAddr(receipt.ContractAddress) + var contractHash string + isDeployment := contract != "" + if isDeployment { + contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String() + // if tx is a contract deployment, publish the data (code) + // codec doesn't matter in this case sine we are not interested in the cid and the db key is multihash-derived + // TODO: THE DATA IS NOT DIRECTLY THE CONTRACT CODE; THERE IS A MISSING PROCESSING STEP HERE + // the contractHash => contract code is not currently correct + if _, err := shared.PublishRaw(tx, ipld.MEthStorageTrie, multihash.KECCAK_256, trx.Data()); err != nil { + return err + } + } + // index tx first so that the receipt can reference it by FK + txModel := models.TxModel{ + Dst: shared.HandleZeroAddrPointer(trx.To()), + Src: shared.HandleZeroAddr(from), + TxHash: trx.Hash().String(), + Index: int64(i), + Data: trx.Data(), + Deployment: isDeployment, + CID: txNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(txNode.Cid()), + } + txID, err := sdt.indexer.indexTransactionCID(tx, txModel, args.headerID) + if err != nil { + return err + } + // index the receipt + rctModel := models.ReceiptModel{ + Topic0s: topicSets[0], + Topic1s: topicSets[1], + Topic2s: topicSets[2], + Topic3s: topicSets[3], + Contract: contract, + ContractHash: contractHash, + LogContracts: logContracts, + CID: rctNode.Cid().String(), + MhKey: shared.MultihashKeyFromCID(rctNode.Cid()), + } + if err := sdt.indexer.indexReceiptCID(tx, rctModel, txID); err != nil { + return err + } + } + return nil +} + +func (sdt *StateDiffIndexer) PushStateNode(tx *BlockTx, stateNode sdtypes.StateNode) error { + // publish the state node + stateCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) + if err != nil { + return err + } + mhKey, _ := shared.MultihashKeyFromCIDString(stateCIDStr) + stateModel := models.StateNodeModel{ + Path: stateNode.Path, + StateKey: common.BytesToHash(stateNode.LeafKey).String(), + CID: stateCIDStr, + MhKey: mhKey, + NodeType: ResolveFromNodeType(stateNode.NodeType), + } + // index the state node, collect the stateID to reference by FK + stateID, err := sdt.indexer.indexStateCID(tx.dbtx, stateModel, tx.headerID) + if err != nil { + return err + } + // if we have a leaf, decode and index the account data + if stateNode.NodeType == sdtypes.Leaf { + var i []interface{} + if err := rlp.DecodeBytes(stateNode.NodeValue, &i); err != nil { + return fmt.Errorf("error decoding state leaf node rlp: %s", err.Error()) + } + if len(i) != 2 { + return fmt.Errorf("eth IPLDPublisher expected state leaf node rlp to decode into two elements") + } + var account state.Account + if err := rlp.DecodeBytes(i[1].([]byte), &account); err != nil { + return fmt.Errorf("error decoding state account rlp: %s", err.Error()) + } + accountModel := models.StateAccountModel{ + Balance: account.Balance.String(), + Nonce: account.Nonce, + CodeHash: account.CodeHash, + StorageRoot: account.Root.String(), + } + if err := sdt.indexer.indexStateAccount(tx.dbtx, accountModel, stateID); err != nil { + return err + } + } + // if there are any storage nodes associated with this node, publish and index them + for _, storageNode := range stateNode.StorageNodes { + storageCIDStr, err := shared.PublishRaw(tx.dbtx, ipld.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) + if err != nil { + return err + } + mhKey, _ := shared.MultihashKeyFromCIDString(storageCIDStr) + storageModel := models.StorageNodeModel{ + Path: storageNode.Path, + StorageKey: common.BytesToHash(storageNode.LeafKey).String(), + CID: storageCIDStr, + MhKey: mhKey, + NodeType: ResolveFromNodeType(storageNode.NodeType), + } + if err := sdt.indexer.indexStorageCID(tx.dbtx, storageModel, stateID); err != nil { + return err + } + } + + return nil +} diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go new file mode 100644 index 000000000000..76092ff198e6 --- /dev/null +++ b/statediff/indexer/indexer_test.go @@ -0,0 +1,243 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-ds-help" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + ind "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + eth "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +func TestShared(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Indexer tests") +} + +var _ = Describe("PublishAndIndexer", func() { + var ( + db *postgres.DB + err error + indexer *ind.StateDiffIndexer + ipfsPgGet = `SELECT data FROM public.blocks + WHERE key = $1` + ) + BeforeEach(func() { + db, err = shared.SetupDB() + Expect(err).ToNot(HaveOccurred()) + indexer = ind.NewStateDiffIndexer(params.MainnetChainConfig, db) + tx, err := indexer.PushBlock( + mocks.MockBlock, + mocks.MockReceipts, + mocks.MockBlock.Difficulty()) + defer tx.Close() + Expect(err).ToNot(HaveOccurred()) + for _, node := range mocks.StateDiffs { + err = indexer.PushStateNode(tx, node) + Expect(err).ToNot(HaveOccurred()) + } + Expect(tx.BlockNumber).To(Equal(mocks.BlockNumber.Uint64())) + }) + AfterEach(func() { + ind.TearDownDB(db) + }) + + Describe("Publish", func() { + It("Publishes and indexes header IPLDs in a single tx", func() { + pgStr := `SELECT cid, td, reward, id + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + ID int + } + header := new(res) + err = db.QueryRowx(pgStr, 1).StructScan(header) + Expect(err).ToNot(HaveOccurred()) + Expect(header.CID).To(Equal(mocks.HeaderCID.String())) + Expect(header.TD).To(Equal(mocks.MockBlock.Difficulty().String())) + Expect(header.Reward).To(Equal("5000000000000011250")) + dc, err := cid.Decode(header.CID) + Expect(err).ToNot(HaveOccurred()) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal(mocks.MockHeaderRlp)) + }) + + It("Publishes and indexes transaction IPLDs in a single tx", func() { + // check that txs were properly indexed + trxs := make([]string, 0) + pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id) + WHERE header_cids.block_number = $1` + err = db.Select(&trxs, pgStr, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(trxs)).To(Equal(3)) + Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue()) + Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue()) + Expect(shared.ListContainsString(trxs, mocks.Trx3CID.String())).To(BeTrue()) + // and published + for _, c := range trxs { + dc, err := cid.Decode(c) + Expect(err).ToNot(HaveOccurred()) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + Expect(err).ToNot(HaveOccurred()) + switch c { + case mocks.Trx1CID.String(): + Expect(data).To(Equal(mocks.MockTransactions.GetRlp(0))) + case mocks.Trx2CID.String(): + Expect(data).To(Equal(mocks.MockTransactions.GetRlp(1))) + case mocks.Trx3CID.String(): + Expect(data).To(Equal(mocks.MockTransactions.GetRlp(2))) + } + } + }) + + It("Publishes and indexes receipt IPLDs in a single tx", func() { + // check receipts were properly indexed + rcts := make([]string, 0) + pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids + WHERE receipt_cids.tx_id = transaction_cids.id + AND transaction_cids.header_id = header_cids.id + AND header_cids.block_number = $1` + err = db.Select(&rcts, pgStr, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(rcts)).To(Equal(3)) + Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue()) + Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue()) + Expect(shared.ListContainsString(rcts, mocks.Rct3CID.String())).To(BeTrue()) + // and published + for _, c := range rcts { + dc, err := cid.Decode(c) + Expect(err).ToNot(HaveOccurred()) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + var data []byte + err = db.Get(&data, ipfsPgGet, prefixedKey) + Expect(err).ToNot(HaveOccurred()) + switch c { + case mocks.Rct1CID.String(): + Expect(data).To(Equal(mocks.MockReceipts.GetRlp(0))) + case mocks.Rct2CID.String(): + Expect(data).To(Equal(mocks.MockReceipts.GetRlp(1))) + case mocks.Rct3CID.String(): + Expect(data).To(Equal(mocks.MockReceipts.GetRlp(2))) + } + } + }) + + It("Publishes and indexes state IPLDs in a single tx", func() { + // check that state nodes were properly indexed and published + stateNodes := make([]eth.StateNodeModel, 0) + pgStr := `SELECT state_cids.id, state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id + FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id) + WHERE header_cids.block_number = $1` + err = db.Select(&stateNodes, pgStr, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(stateNodes)).To(Equal(2)) + for _, stateNode := range stateNodes { + var data []byte + dc, err := cid.Decode(stateNode.CID) + Expect(err).ToNot(HaveOccurred()) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(&data, ipfsPgGet, prefixedKey) + Expect(err).ToNot(HaveOccurred()) + pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` + var account eth.StateAccountModel + err = db.Get(&account, pgStr, stateNode.ID) + Expect(err).ToNot(HaveOccurred()) + if stateNode.CID == mocks.State1CID.String() { + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex())) + Expect(stateNode.Path).To(Equal([]byte{'\x06'})) + Expect(data).To(Equal(mocks.ContractLeafNode)) + Expect(account).To(Equal(eth.StateAccountModel{ + ID: account.ID, + StateID: stateNode.ID, + Balance: "0", + CodeHash: mocks.ContractCodeHash.Bytes(), + StorageRoot: mocks.ContractRoot, + Nonce: 1, + })) + } + if stateNode.CID == mocks.State2CID.String() { + Expect(stateNode.NodeType).To(Equal(2)) + Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex())) + Expect(stateNode.Path).To(Equal([]byte{'\x0c'})) + Expect(data).To(Equal(mocks.AccountLeafNode)) + Expect(account).To(Equal(eth.StateAccountModel{ + ID: account.ID, + StateID: stateNode.ID, + Balance: "1000", + CodeHash: mocks.AccountCodeHash.Bytes(), + StorageRoot: mocks.AccountRoot, + Nonce: 0, + })) + } + } + pgStr = `SELECT * from eth.state_accounts WHERE state_id = $1` + }) + + It("Publishes and indexes storage IPLDs in a single tx", func() { + // check that storage nodes were properly indexed + storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0) + pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path + FROM eth.storage_cids, eth.state_cids, eth.header_cids + WHERE storage_cids.state_id = state_cids.id + AND state_cids.header_id = header_cids.id + AND header_cids.block_number = $1` + err = db.Select(&storageNodes, pgStr, 1) + Expect(err).ToNot(HaveOccurred()) + Expect(len(storageNodes)).To(Equal(1)) + Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{ + CID: mocks.StorageCID.String(), + NodeType: 2, + StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), + StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), + Path: []byte{}, + })) + var data []byte + dc, err := cid.Decode(storageNodes[0].CID) + Expect(err).ToNot(HaveOccurred()) + mhKey := dshelp.MultihashToDsKey(dc.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() + err = db.Get(&data, ipfsPgGet, prefixedKey) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal(mocks.StorageLeafNode)) + }) + }) +}) diff --git a/statediff/indexer/ipfs/ipld/eth_account.go b/statediff/indexer/ipfs/ipld/eth_account.go new file mode 100644 index 000000000000..5d80af1d9e28 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_account.go @@ -0,0 +1,175 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "math/big" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// EthAccountSnapshot (eth-account-snapshot codec 0x97) +// represents an ethereum account, i.e. a wallet address or +// a smart contract +type EthAccountSnapshot struct { + *EthAccount + + cid cid.Cid + rawdata []byte +} + +// EthAccount is the building block of EthAccountSnapshot. +// Or, is the former stripped of its cid and rawdata components. +type EthAccount struct { + Nonce uint64 + Balance *big.Int + Root []byte // This is the storage root trie + CodeHash []byte // This is the hash of the EVM code +} + +// Static (compile time) check that EthAccountSnapshot satisfies the +// node.Node interface. +var _ node.Node = (*EthAccountSnapshot)(nil) + +/* + INPUT +*/ + +// Input should be managed by EthStateTrie + +/* + OUTPUT +*/ + +// Output should be managed by EthStateTrie + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the account snapshot. +func (as *EthAccountSnapshot) RawData() []byte { + return as.rawdata +} + +// Cid returns the cid of the transaction. +func (as *EthAccountSnapshot) Cid() cid.Cid { + return as.cid +} + +// String is a helper for output +func (as *EthAccountSnapshot) String() string { + return fmt.Sprintf("", as.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (as *EthAccountSnapshot) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-account-snapshot", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (as *EthAccountSnapshot) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return as, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + case "balance": + return as.Balance, nil, nil + case "codeHash": + return &node.Link{Cid: keccak256ToCid(RawBinary, as.CodeHash)}, nil, nil + case "nonce": + return as.Nonce, nil, nil + case "root": + return &node.Link{Cid: keccak256ToCid(MEthStorageTrie, as.Root)}, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (as *EthAccountSnapshot) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"balance", "codeHash", "nonce", "root"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (as *EthAccountSnapshot) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := as.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (as *EthAccountSnapshot) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (as *EthAccountSnapshot) Size() (uint64, error) { + return 0, nil +} + +/* + EthAccountSnapshot functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (as *EthAccountSnapshot) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "balance": as.Balance, + "codeHash": keccak256ToCid(RawBinary, as.CodeHash), + "nonce": as.Nonce, + "root": keccak256ToCid(MEthStorageTrie, as.Root), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_header.go b/statediff/indexer/ipfs/ipld/eth_header.go new file mode 100644 index 000000000000..c33931d4fde5 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_header.go @@ -0,0 +1,256 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthHeader (eth-block, codec 0x90), represents an ethereum block header +type EthHeader struct { + *types.Header + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthHeader satisfies the node.Node interface. +var _ node.Node = (*EthHeader)(nil) + +/* + INPUT +*/ + +// NewEthHeader converts a *types.Header into an EthHeader IPLD node +func NewEthHeader(header *types.Header) (*EthHeader, error) { + headerRLP, err := rlp.EncodeToBytes(header) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthHeader, headerRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthHeader{ + Header: header, + cid: c, + rawdata: headerRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthHeader takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthHeader(c cid.Cid, b []byte) (*EthHeader, error) { + var h *types.Header + if err := rlp.DecodeBytes(b, h); err != nil { + return nil, err + } + return &EthHeader{ + Header: h, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the block header. +func (b *EthHeader) RawData() []byte { + return b.rawdata +} + +// Cid returns the cid of the block header. +func (b *EthHeader) Cid() cid.Cid { + return b.cid +} + +// String is a helper for output +func (b *EthHeader) String() string { + return fmt.Sprintf("", b.cid) +} + +// Loggable returns a map the type of IPLD Link. +func (b *EthHeader) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-block", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (b *EthHeader) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return b, nil, nil + } + + first, rest := p[0], p[1:] + + switch first { + case "parent": + return &node.Link{Cid: commonHashToCid(MEthHeader, b.ParentHash)}, rest, nil + case "receipts": + return &node.Link{Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, rest, nil + case "root": + return &node.Link{Cid: commonHashToCid(MEthStateTrie, b.Root)}, rest, nil + case "tx": + return &node.Link{Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, rest, nil + case "uncles": + return &node.Link{Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, rest, nil + } + + if len(p) != 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", first) + } + + switch first { + case "bloom": + return b.Bloom, nil, nil + case "coinbase": + return b.Coinbase, nil, nil + case "difficulty": + return b.Difficulty, nil, nil + case "extra": + // This is a []byte. By default they are marshalled into Base64. + return fmt.Sprintf("0x%x", b.Extra), nil, nil + case "gaslimit": + return b.GasLimit, nil, nil + case "gasused": + return b.GasUsed, nil, nil + case "mixdigest": + return b.MixDigest, nil, nil + case "nonce": + return b.Nonce, nil, nil + case "number": + return b.Number, nil, nil + case "time": + return b.Time, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (b *EthHeader) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + return []string{ + "time", + "bloom", + "coinbase", + "difficulty", + "extra", + "gaslimit", + "gasused", + "mixdigest", + "nonce", + "number", + "parent", + "receipts", + "root", + "tx", + "uncles", + } +} + +// ResolveLink is a helper function that allows easier traversal of links through blocks +func (b *EthHeader) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := b.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (b *EthHeader) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +// HINT: Use `ipfs refs ` +func (b *EthHeader) Links() []*node.Link { + return []*node.Link{ + {Cid: commonHashToCid(MEthHeader, b.ParentHash)}, + {Cid: commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash)}, + {Cid: commonHashToCid(MEthStateTrie, b.Root)}, + {Cid: commonHashToCid(MEthTxTrie, b.TxHash)}, + {Cid: commonHashToCid(MEthHeaderList, b.UncleHash)}, + } +} + +// Stat will go away. It is here to comply with the Node interface. +func (b *EthHeader) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the Node interface. +func (b *EthHeader) Size() (uint64, error) { + return 0, nil +} + +/* + EthHeader functions +*/ + +// MarshalJSON processes the block header into readable JSON format, +// converting the right links into their cids, and keeping the original +// hex hash, allowing the user to simplify external queries. +func (b *EthHeader) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "time": b.Time, + "bloom": b.Bloom, + "coinbase": b.Coinbase, + "difficulty": b.Difficulty, + "extra": fmt.Sprintf("0x%x", b.Extra), + "gaslimit": b.GasLimit, + "gasused": b.GasUsed, + "mixdigest": b.MixDigest, + "nonce": b.Nonce, + "number": b.Number, + "parent": commonHashToCid(MEthHeader, b.ParentHash), + "receipts": commonHashToCid(MEthTxReceiptTrie, b.ReceiptHash), + "root": commonHashToCid(MEthStateTrie, b.Root), + "tx": commonHashToCid(MEthTxTrie, b.TxHash), + "uncles": commonHashToCid(MEthHeaderList, b.UncleHash), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_parser.go b/statediff/indexer/ipfs/ipld/eth_parser.go new file mode 100644 index 000000000000..f02d7d401f91 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_parser.go @@ -0,0 +1,97 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/core/types" +) + +// FromBlockAndReceipts takes a block and processes it +// to return it a set of IPLD nodes for further processing. +func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, error) { + // Process the header + headerNode, err := NewEthHeader(block.Header()) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + // Process the uncles + uncleNodes := make([]*EthHeader, len(block.Uncles())) + for i, uncle := range block.Uncles() { + uncleNode, err := NewEthHeader(uncle) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + uncleNodes[i] = uncleNode + } + // Process the txs + ethTxNodes, ethTxTrieNodes, err := processTransactions(block.Transactions(), + block.Header().TxHash[:]) + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + // Process the receipts + ethRctNodes, ethRctTrieNodes, err := processReceipts(receipts, + block.Header().ReceiptHash[:]) + return headerNode, uncleNodes, ethTxNodes, ethTxTrieNodes, ethRctNodes, ethRctTrieNodes, err +} + +// processTransactions will take the found transactions in a parsed block body +// to return IPLD node slices for eth-tx and eth-tx-trie +func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) { + var ethTxNodes []*EthTx + transactionTrie := newTxTrie() + + for idx, tx := range txs { + ethTx, err := NewEthTx(tx) + if err != nil { + return nil, nil, err + } + ethTxNodes = append(ethTxNodes, ethTx) + transactionTrie.add(idx, ethTx.RawData()) + } + + if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) { + return nil, nil, fmt.Errorf("wrong transaction hash computed") + } + + return ethTxNodes, transactionTrie.getNodes(), nil +} + +// processReceipts will take in receipts +// to return IPLD node slices for eth-rct and eth-rct-trie +func processReceipts(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, error) { + var ethRctNodes []*EthReceipt + receiptTrie := newRctTrie() + + for idx, rct := range rcts { + ethRct, err := NewReceipt(rct) + if err != nil { + return nil, nil, err + } + ethRctNodes = append(ethRctNodes, ethRct) + receiptTrie.add(idx, ethRct.RawData()) + } + + if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) { + return nil, nil, fmt.Errorf("wrong receipt hash computed") + } + + return ethRctNodes, receiptTrie.getNodes(), nil +} diff --git a/statediff/indexer/ipfs/ipld/eth_receipt.go b/statediff/indexer/ipfs/ipld/eth_receipt.go new file mode 100644 index 000000000000..cfa46b36e601 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_receipt.go @@ -0,0 +1,199 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +type EthReceipt struct { + *types.Receipt + + rawdata []byte + cid cid.Cid +} + +// Static (compile time) check that EthReceipt satisfies the node.Node interface. +var _ node.Node = (*EthReceipt)(nil) + +/* + INPUT +*/ + +// NewReceipt converts a types.ReceiptForStorage to an EthReceipt IPLD node +func NewReceipt(receipt *types.Receipt) (*EthReceipt, error) { + receiptRLP, err := rlp.EncodeToBytes(receipt) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, receiptRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: receipt, + cid: c, + rawdata: receiptRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthReceipt takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthReceipt(c cid.Cid, b []byte) (*EthReceipt, error) { + var r *types.Receipt + if err := rlp.DecodeBytes(b, r); err != nil { + return nil, err + } + return &EthReceipt{ + Receipt: r, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +func (node *EthReceipt) RawData() []byte { + return node.rawdata +} + +func (node *EthReceipt) Cid() cid.Cid { + return node.cid +} + +// String is a helper for output +func (r *EthReceipt) String() string { + return fmt.Sprintf("", r.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (r *EthReceipt) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-receipt", + } +} + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (r *EthReceipt) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return r, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "root": + return r.PostState, nil, nil + case "status": + return r.Status, nil, nil + case "cumulativeGasUsed": + return r.CumulativeGasUsed, nil, nil + case "logsBloom": + return r.Bloom, nil, nil + case "logs": + return r.Logs, nil, nil + case "transactionHash": + return r.TxHash, nil, nil + case "contractAddress": + return r.ContractAddress, nil, nil + case "gasUsed": + return r.GasUsed, nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (r *EthReceipt) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"root", "status", "cumulativeGasUsed", "logsBloom", "logs", "transactionHash", "contractAddress", "gasUsed"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (r *EthReceipt) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := r.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the Node interface. +func (*EthReceipt) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (*EthReceipt) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (r *EthReceipt) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (r *EthReceipt) Size() (uint64, error) { + return strconv.ParseUint(r.Receipt.Size().String(), 10, 64) +} + +/* + EthReceipt functions +*/ + +// MarshalJSON processes the receipt into readable JSON format. +func (r *EthReceipt) MarshalJSON() ([]byte, error) { + out := map[string]interface{}{ + "root": r.PostState, + "status": r.Status, + "cumulativeGasUsed": r.CumulativeGasUsed, + "logsBloom": r.Bloom, + "logs": r.Logs, + "transactionHash": r.TxHash, + "contractAddress": r.ContractAddress, + "gasUsed": r.GasUsed, + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go new file mode 100644 index 000000000000..6a1b7e4068f8 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go @@ -0,0 +1,152 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthRctTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthRctTrie struct { + *TrieNode +} + +// Static (compile time) check that EthRctTrie satisfies the node.Node interface. +var _ node.Node = (*EthRctTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthRctTrie returns an EthRctTrie object from its cid and rawdata. +func DecodeEthRctTrie(c cid.Cid, b []byte) (*EthRctTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthRctTrieLeaf) + if err != nil { + return nil, err + } + return &EthRctTrie{TrieNode: tn}, nil +} + +// decodeEthRctTrieLeaf parses a eth-rct-trie leaf +//from decoded RLP elements +func decodeEthRctTrieLeaf(i []interface{}) ([]interface{}, error) { + var r types.Receipt + err := rlp.DecodeBytes(i[1].([]byte), &r) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTxReceipt, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthReceipt{ + Receipt: &r, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthRctTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthRctTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthRctTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthRctTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-rct-trie", + } +} + +/* + EthRctTrie functions +*/ + +// rctTrie wraps a localTrie for use on the receipt trie. +type rctTrie struct { + *localTrie +} + +// newRctTrie initializes and returns a rctTrie. +func newRctTrie() *rctTrie { + return &rctTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its database keys, to return a slice +// of EthRctTrie nodes. +func (rt *rctTrie) getNodes() []*EthRctTrie { + keys := rt.getKeys() + var out []*EthRctTrie + it := rt.trie.NodeIterator([]byte{}) + for it.Next(true) { + + } + for _, k := range keys { + rawdata, err := rt.db.Get(k) + if err != nil { + panic(err) + } + c, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256) + if err != nil { + return nil + } + tn := &TrieNode{ + cid: c, + rawdata: rawdata, + } + out = append(out, &EthRctTrie{TrieNode: tn}) + } + + return out +} diff --git a/statediff/indexer/ipfs/ipld/eth_state.go b/statediff/indexer/ipfs/ipld/eth_state.go new file mode 100644 index 000000000000..a127f956ebe9 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_state.go @@ -0,0 +1,114 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/rlp" +) + +// EthStateTrie (eth-state-trie, codec 0x96), represents +// a node from the satte trie in ethereum. +type EthStateTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStateTrie satisfies the node.Node interface. +var _ node.Node = (*EthStateTrie)(nil) + +/* + INPUT +*/ + +// FromStateTrieRLP takes the RLP representation of an ethereum +// state trie node to return it as an IPLD node for further processing. +func FromStateTrieRLP(raw []byte) (*EthStateTrie, error) { + c, err := RawdataToCid(MEthStateTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStateTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStateTrie returns an EthStateTrie object from its cid and rawdata. +func DecodeEthStateTrie(c cid.Cid, b []byte) (*EthStateTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStateTrieLeaf) + if err != nil { + return nil, err + } + return &EthStateTrie{TrieNode: tn}, nil +} + +// decodeEthStateTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStateTrieLeaf(i []interface{}) ([]interface{}, error) { + var account EthAccount + err := rlp.DecodeBytes(i[1].([]byte), &account) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthAccountSnapshot, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthAccountSnapshot{ + EthAccount: &account, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the state trie node. +func (st *EthStateTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the state trie node. +func (st *EthStateTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStateTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStateTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-state-trie", + } +} diff --git a/statediff/indexer/ipfs/ipld/eth_storage.go b/statediff/indexer/ipfs/ipld/eth_storage.go new file mode 100644 index 000000000000..779cad4d933e --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_storage.go @@ -0,0 +1,100 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" +) + +// EthStorageTrie (eth-storage-trie, codec 0x98), represents +// a node from the storage trie in ethereum. +type EthStorageTrie struct { + *TrieNode +} + +// Static (compile time) check that EthStorageTrie satisfies the node.Node interface. +var _ node.Node = (*EthStorageTrie)(nil) + +/* + INPUT +*/ + +// FromStorageTrieRLP takes the RLP representation of an ethereum +// storage trie node to return it as an IPLD node for further processing. +func FromStorageTrieRLP(raw []byte) (*EthStorageTrie, error) { + c, err := RawdataToCid(MEthStorageTrie, raw, multihash.KECCAK_256) + if err != nil { + return nil, err + } + + // Let's run the whole mile and process the nodeKind and + // its elements, in case somebody would need this function + // to parse an RLP element from the filesystem + return DecodeEthStorageTrie(c, raw) +} + +/* + OUTPUT +*/ + +// DecodeEthStorageTrie returns an EthStorageTrie object from its cid and rawdata. +func DecodeEthStorageTrie(c cid.Cid, b []byte) (*EthStorageTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthStorageTrieLeaf) + if err != nil { + return nil, err + } + return &EthStorageTrie{TrieNode: tn}, nil +} + +// decodeEthStorageTrieLeaf parses a eth-tx-trie leaf +// from decoded RLP elements +func decodeEthStorageTrieLeaf(i []interface{}) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + i[1].([]byte), + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the storage trie node. +func (st *EthStorageTrie) RawData() []byte { + return st.rawdata +} + +// Cid returns the cid of the storage trie node. +func (st *EthStorageTrie) Cid() cid.Cid { + return st.cid +} + +// String is a helper for output +func (st *EthStorageTrie) String() string { + return fmt.Sprintf("", st.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (st *EthStorageTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-storage-trie", + } +} diff --git a/statediff/indexer/ipfs/ipld/eth_tx.go b/statediff/indexer/ipfs/ipld/eth_tx.go new file mode 100644 index 000000000000..4fc4d20a6a6d --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_tx.go @@ -0,0 +1,215 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + mh "github.com/multiformats/go-multihash" +) + +// EthTx (eth-tx codec 0x93) represents an ethereum transaction +type EthTx struct { + *types.Transaction + + cid cid.Cid + rawdata []byte +} + +// Static (compile time) check that EthTx satisfies the node.Node interface. +var _ node.Node = (*EthTx)(nil) + +/* + INPUT +*/ + +// NewEthTx converts a *types.Transaction to an EthTx IPLD node +func NewEthTx(tx *types.Transaction) (*EthTx, error) { + txRLP, err := rlp.EncodeToBytes(tx) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, txRLP, mh.KECCAK_256) + if err != nil { + return nil, err + } + return &EthTx{ + Transaction: tx, + cid: c, + rawdata: txRLP, + }, nil +} + +/* + OUTPUT +*/ + +// DecodeEthTx takes a cid and its raw binary data +// from IPFS and returns an EthTx object for further processing. +func DecodeEthTx(c cid.Cid, b []byte) (*EthTx, error) { + var t *types.Transaction + if err := rlp.DecodeBytes(b, t); err != nil { + return nil, err + } + return &EthTx{ + Transaction: t, + cid: c, + rawdata: b, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTx) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTx) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTx) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTx) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx", + } +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *EthTx) Resolve(p []string) (interface{}, []string, error) { + if len(p) == 0 { + return t, nil, nil + } + + if len(p) > 1 { + return nil, nil, fmt.Errorf("unexpected path elements past %s", p[0]) + } + + switch p[0] { + + case "gas": + return t.Gas(), nil, nil + case "gasPrice": + return t.GasPrice(), nil, nil + case "input": + return fmt.Sprintf("%x", t.Data()), nil, nil + case "nonce": + return t.Nonce(), nil, nil + case "r": + _, r, _ := t.RawSignatureValues() + return hexutil.EncodeBig(r), nil, nil + case "s": + _, _, s := t.RawSignatureValues() + return hexutil.EncodeBig(s), nil, nil + case "toAddress": + return t.To(), nil, nil + case "v": + v, _, _ := t.RawSignatureValues() + return hexutil.EncodeBig(v), nil, nil + case "value": + return hexutil.EncodeBig(t.Value()), nil, nil + default: + return nil, nil, fmt.Errorf("no such link") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *EthTx) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + return []string{"gas", "gasPrice", "input", "nonce", "r", "s", "toAddress", "v", "value"} +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *EthTx) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + if lnk, ok := obj.(*node.Link); ok { + return lnk, rest, nil + } + + return nil, nil, fmt.Errorf("resolved item was not a link") +} + +// Copy will go away. It is here to comply with the interface. +func (t *EthTx) Copy() node.Node { + panic("implement me") +} + +// Links is a helper function that returns all links within this object +func (t *EthTx) Links() []*node.Link { + return nil +} + +// Stat will go away. It is here to comply with the interface. +func (t *EthTx) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *EthTx) Size() (uint64, error) { + return strconv.ParseUint(t.Transaction.Size().String(), 10, 64) +} + +/* + EthTx functions +*/ + +// MarshalJSON processes the transaction into readable JSON format. +func (t *EthTx) MarshalJSON() ([]byte, error) { + v, r, s := t.RawSignatureValues() + + out := map[string]interface{}{ + "gas": t.Gas(), + "gasPrice": hexutil.EncodeBig(t.GasPrice()), + "input": fmt.Sprintf("%x", t.Data()), + "nonce": t.Nonce(), + "r": hexutil.EncodeBig(r), + "s": hexutil.EncodeBig(s), + "toAddress": t.To(), + "v": hexutil.EncodeBig(v), + "value": hexutil.EncodeBig(t.Value()), + } + return json.Marshal(out) +} diff --git a/statediff/indexer/ipfs/ipld/eth_tx_trie.go b/statediff/indexer/ipfs/ipld/eth_tx_trie.go new file mode 100644 index 000000000000..6f106f6d8f30 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/eth_tx_trie.go @@ -0,0 +1,152 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "fmt" + + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// EthTxTrie (eth-tx-trie codec 0x92) represents +// a node from the transaction trie in ethereum. +type EthTxTrie struct { + *TrieNode +} + +// Static (compile time) check that EthTxTrie satisfies the node.Node interface. +var _ node.Node = (*EthTxTrie)(nil) + +/* + INPUT +*/ + +// To create a proper trie of the eth-tx-trie objects, it is required +// to input all transactions belonging to a forest in a single step. +// We are adding the transactions, and creating its trie on +// block body parsing time. + +/* + OUTPUT +*/ + +// DecodeEthTxTrie returns an EthTxTrie object from its cid and rawdata. +func DecodeEthTxTrie(c cid.Cid, b []byte) (*EthTxTrie, error) { + tn, err := decodeTrieNode(c, b, decodeEthTxTrieLeaf) + if err != nil { + return nil, err + } + return &EthTxTrie{TrieNode: tn}, nil +} + +// decodeEthTxTrieLeaf parses a eth-tx-trie leaf +//from decoded RLP elements +func decodeEthTxTrieLeaf(i []interface{}) ([]interface{}, error) { + var t types.Transaction + err := rlp.DecodeBytes(i[1].([]byte), &t) + if err != nil { + return nil, err + } + c, err := RawdataToCid(MEthTx, i[1].([]byte), multihash.KECCAK_256) + if err != nil { + return nil, err + } + return []interface{}{ + i[0].([]byte), + &EthTx{ + Transaction: &t, + cid: c, + rawdata: i[1].([]byte), + }, + }, nil +} + +/* + Block INTERFACE +*/ + +// RawData returns the binary of the RLP encode of the transaction. +func (t *EthTxTrie) RawData() []byte { + return t.rawdata +} + +// Cid returns the cid of the transaction. +func (t *EthTxTrie) Cid() cid.Cid { + return t.cid +} + +// String is a helper for output +func (t *EthTxTrie) String() string { + return fmt.Sprintf("", t.cid) +} + +// Loggable returns in a map the type of IPLD Link. +func (t *EthTxTrie) Loggable() map[string]interface{} { + return map[string]interface{}{ + "type": "eth-tx-trie", + } +} + +/* + EthTxTrie functions +*/ + +// txTrie wraps a localTrie for use on the transaction trie. +type txTrie struct { + *localTrie +} + +// newTxTrie initializes and returns a txTrie. +func newTxTrie() *txTrie { + return &txTrie{ + localTrie: newLocalTrie(), + } +} + +// getNodes invokes the localTrie, which computes the root hash of the +// transaction trie and returns its database keys, to return a slice +// of EthTxTrie nodes. +func (tt *txTrie) getNodes() []*EthTxTrie { + keys := tt.getKeys() + var out []*EthTxTrie + it := tt.trie.NodeIterator([]byte{}) + for it.Next(true) { + + } + for _, k := range keys { + rawdata, err := tt.db.Get(k) + if err != nil { + panic(err) + } + c, err := RawdataToCid(MEthTxTrie, rawdata, multihash.KECCAK_256) + if err != nil { + return nil + } + tn := &TrieNode{ + cid: c, + rawdata: rawdata, + } + out = append(out, &EthTxTrie{TrieNode: tn}) + } + + return out +} diff --git a/statediff/indexer/ipfs/ipld/shared.go b/statediff/indexer/ipfs/ipld/shared.go new file mode 100644 index 000000000000..cfdd7446add9 --- /dev/null +++ b/statediff/indexer/ipfs/ipld/shared.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +// IPLD Codecs for Ethereum +// See the authoritative document: +// https://github.com/multiformats/multicodec/blob/master/table.csv +const ( + RawBinary = 0x55 + MEthHeader = 0x90 + MEthHeaderList = 0x91 + MEthTxTrie = 0x92 + MEthTx = 0x93 + MEthTxReceiptTrie = 0x94 + MEthTxReceipt = 0x95 + MEthStateTrie = 0x96 + MEthAccountSnapshot = 0x97 + MEthStorageTrie = 0x98 +) + +// RawdataToCid takes the desired codec and a slice of bytes +// and returns the proper cid of the object. +func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) { + c, err := cid.Prefix{ + Codec: codec, + Version: 1, + MhType: multiHash, + MhLength: -1, + }.Sum(rawdata) + if err != nil { + return cid.Cid{}, err + } + return c, nil +} + +// keccak256ToCid takes a keccak256 hash and returns its cid based on +// the codec given. +func keccak256ToCid(codec uint64, h []byte) cid.Cid { + buf, err := mh.Encode(h, mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mh.Multihash(buf)) +} + +// commonHashToCid takes a go-ethereum common.Hash and returns its +// cid based on the codec given, +func commonHashToCid(codec uint64, h common.Hash) cid.Cid { + mhash, err := mh.Encode(h[:], mh.KECCAK_256) + if err != nil { + panic(err) + } + + return cid.NewCidV1(codec, mhash) +} + +// localTrie wraps a go-ethereum trie and its underlying memory db. +// It contributes to the creation of the trie node objects. +type localTrie struct { + keys [][]byte + db ethdb.Database + trie *trie.Trie +} + +// newLocalTrie initializes and returns a localTrie object +func newLocalTrie() *localTrie { + var err error + lt := &localTrie{} + lt.db = rawdb.NewMemoryDatabase() + lt.trie, err = trie.New(common.Hash{}, trie.NewDatabase(lt.db)) + if err != nil { + panic(err) + } + return lt +} + +// add receives the index of an object and its rawdata value +// and includes it into the localTrie +func (lt *localTrie) add(idx int, rawdata []byte) { + key, err := rlp.EncodeToBytes(uint(idx)) + if err != nil { + panic(err) + } + lt.keys = append(lt.keys, key) + if err := lt.db.Put(key, rawdata); err != nil { + panic(err) + } + lt.trie.Update(key, rawdata) +} + +// rootHash returns the computed trie root. +// Useful for sanity checks on parsed data. +func (lt *localTrie) rootHash() []byte { + return lt.trie.Hash().Bytes() +} + +// getKeys returns the stored keys of the memory database +// of the localTrie for further processing. +func (lt *localTrie) getKeys() [][]byte { + return lt.keys +} diff --git a/statediff/indexer/ipfs/ipld/trie_node.go b/statediff/indexer/ipfs/ipld/trie_node.go new file mode 100644 index 000000000000..788f76db84cf --- /dev/null +++ b/statediff/indexer/ipfs/ipld/trie_node.go @@ -0,0 +1,444 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-cid" + node "github.com/ipfs/go-ipld-format" +) + +// TrieNode is the general abstraction for +//ethereum IPLD trie nodes. +type TrieNode struct { + // leaf, extension or branch + nodeKind string + + // If leaf or extension: [0] is key, [1] is val. + // If branch: [0] - [16] are children. + elements []interface{} + + // IPLD block information + cid cid.Cid + rawdata []byte +} + +/* + OUTPUT +*/ + +type trieNodeLeafDecoder func([]interface{}) ([]interface{}, error) + +// decodeTrieNode returns a TrieNode object from an IPLD block's +// cid and rawdata. +func decodeTrieNode(c cid.Cid, b []byte, + leafDecoder trieNodeLeafDecoder) (*TrieNode, error) { + var ( + i, decoded, elements []interface{} + nodeKind string + err error + ) + + if err = rlp.DecodeBytes(b, &i); err != nil { + return nil, err + } + + codec := c.Type() + switch len(i) { + case 2: + nodeKind, decoded, err = decodeCompactKey(i) + if err != nil { + return nil, err + } + + if nodeKind == "extension" { + elements, err = parseTrieNodeExtension(decoded, codec) + } + if nodeKind == "leaf" { + elements, err = leafDecoder(decoded) + } + if nodeKind != "extension" && nodeKind != "leaf" { + return nil, fmt.Errorf("unexpected nodeKind returned from decoder") + } + case 17: + nodeKind = "branch" + elements, err = parseTrieNodeBranch(i, codec) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown trie node type") + } + + return &TrieNode{ + nodeKind: nodeKind, + elements: elements, + rawdata: b, + cid: c, + }, nil +} + +// decodeCompactKey takes a compact key, and returns its nodeKind and value. +func decodeCompactKey(i []interface{}) (string, []interface{}, error) { + first := i[0].([]byte) + last := i[1].([]byte) + + switch first[0] / 16 { + case '\x00': + return "extension", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x01': + return "extension", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + case '\x02': + return "leaf", []interface{}{ + nibbleToByte(first)[2:], + last, + }, nil + case '\x03': + return "leaf", []interface{}{ + nibbleToByte(first)[1:], + last, + }, nil + default: + return "", nil, fmt.Errorf("unknown hex prefix") + } +} + +// parseTrieNodeExtension helper improves readability +func parseTrieNodeExtension(i []interface{}, codec uint64) ([]interface{}, error) { + return []interface{}{ + i[0].([]byte), + keccak256ToCid(codec, i[1].([]byte)), + }, nil +} + +// parseTrieNodeBranch helper improves readability +func parseTrieNodeBranch(i []interface{}, codec uint64) ([]interface{}, error) { + var out []interface{} + + for i, vi := range i { + v, ok := vi.([]byte) + // Sometimes this throws "panic: interface conversion: interface {} is []interface {}, not []uint8" + // Figure out why, and if it is okay to continue + if !ok { + return nil, fmt.Errorf("unable to decode branch node entry into []byte at position: %d value: %+v", i, vi) + } + + switch len(v) { + case 0: + out = append(out, nil) + case 32: + out = append(out, keccak256ToCid(codec, v)) + default: + return nil, fmt.Errorf("unrecognized object: %v", v) + } + } + + return out, nil +} + +/* + Node INTERFACE +*/ + +// Resolve resolves a path through this node, stopping at any link boundary +// and returning the object found as well as the remaining path to traverse +func (t *TrieNode) Resolve(p []string) (interface{}, []string, error) { + switch t.nodeKind { + case "extension": + return t.resolveTrieNodeExtension(p) + case "leaf": + return t.resolveTrieNodeLeaf(p) + case "branch": + return t.resolveTrieNodeBranch(p) + default: + return nil, nil, fmt.Errorf("nodeKind case not implemented") + } +} + +// Tree lists all paths within the object under 'path', and up to the given depth. +// To list the entire object (similar to `find .`) pass "" and -1 +func (t *TrieNode) Tree(p string, depth int) []string { + if p != "" || depth == 0 { + return nil + } + + var out []string + + switch t.nodeKind { + case "extension": + var val string + for _, e := range t.elements[0].([]byte) { + val += fmt.Sprintf("%x", e) + } + return []string{val} + case "branch": + for i, elem := range t.elements { + if _, ok := elem.(*cid.Cid); ok { + out = append(out, fmt.Sprintf("%x", i)) + } + } + return out + + default: + return nil + } +} + +// ResolveLink is a helper function that calls resolve and asserts the +// output is a link +func (t *TrieNode) ResolveLink(p []string) (*node.Link, []string, error) { + obj, rest, err := t.Resolve(p) + if err != nil { + return nil, nil, err + } + + lnk, ok := obj.(*node.Link) + if !ok { + return nil, nil, fmt.Errorf("was not a link") + } + + return lnk, rest, nil +} + +// Copy will go away. It is here to comply with the interface. +func (t *TrieNode) Copy() node.Node { + panic("dont use this yet") +} + +// Links is a helper function that returns all links within this object +func (t *TrieNode) Links() []*node.Link { + var out []*node.Link + + for _, i := range t.elements { + c, ok := i.(cid.Cid) + if ok { + out = append(out, &node.Link{Cid: c}) + } + } + + return out +} + +// Stat will go away. It is here to comply with the interface. +func (t *TrieNode) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +// Size will go away. It is here to comply with the interface. +func (t *TrieNode) Size() (uint64, error) { + return 0, nil +} + +/* + TrieNode functions +*/ + +// MarshalJSON processes the transaction trie into readable JSON format. +func (t *TrieNode) MarshalJSON() ([]byte, error) { + var out map[string]interface{} + + switch t.nodeKind { + case "extension": + fallthrough + case "leaf": + var hexPrefix string + for _, e := range t.elements[0].([]byte) { + hexPrefix += fmt.Sprintf("%x", e) + } + + // if we got a byte we need to do this casting otherwise + // it will be marshaled to a base64 encoded value + if _, ok := t.elements[1].([]byte); ok { + var hexVal string + for _, e := range t.elements[1].([]byte) { + hexVal += fmt.Sprintf("%x", e) + } + + t.elements[1] = hexVal + } + + out = map[string]interface{}{ + "type": t.nodeKind, + hexPrefix: t.elements[1], + } + + case "branch": + out = map[string]interface{}{ + "type": "branch", + "0": t.elements[0], + "1": t.elements[1], + "2": t.elements[2], + "3": t.elements[3], + "4": t.elements[4], + "5": t.elements[5], + "6": t.elements[6], + "7": t.elements[7], + "8": t.elements[8], + "9": t.elements[9], + "a": t.elements[10], + "b": t.elements[11], + "c": t.elements[12], + "d": t.elements[13], + "e": t.elements[14], + "f": t.elements[15], + } + default: + return nil, fmt.Errorf("nodeKind %s not supported", t.nodeKind) + } + + return json.Marshal(out) +} + +// nibbleToByte expands the nibbles of a byte slice into their own bytes. +func nibbleToByte(k []byte) []byte { + var out []byte + + for _, b := range k { + out = append(out, b/16) + out = append(out, b%16) + } + + return out +} + +// Resolve reading conveniences +func (t *TrieNode) resolveTrieNodeExtension(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this extension") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + return &node.Link{Cid: t.elements[1].(cid.Cid)}, rest, nil +} + +func (t *TrieNode) resolveTrieNodeLeaf(p []string) (interface{}, []string, error) { + nibbles := t.elements[0].([]byte) + + if len(nibbles) != 0 { + idx, rest := shiftFromPath(p, len(nibbles)) + if len(idx) < len(nibbles) { + return nil, nil, fmt.Errorf("not enough nibbles to traverse this leaf") + } + + for _, i := range idx { + if getHexIndex(string(i)) == -1 { + return nil, nil, fmt.Errorf("invalid path element") + } + } + + for i, n := range nibbles { + if string(idx[i]) != fmt.Sprintf("%x", n) { + return nil, nil, fmt.Errorf("no such link in this extension") + } + } + + p = rest + } + + link, ok := t.elements[1].(node.Node) + if !ok { + return nil, nil, fmt.Errorf("leaf children is not an IPLD node") + } + + return link.Resolve(p) +} + +func (t *TrieNode) resolveTrieNodeBranch(p []string) (interface{}, []string, error) { + idx, rest := shiftFromPath(p, 1) + hidx := getHexIndex(idx) + if hidx == -1 { + return nil, nil, fmt.Errorf("incorrect path") + } + + child := t.elements[hidx] + if child != nil { + return &node.Link{Cid: child.(cid.Cid)}, rest, nil + } + return nil, nil, fmt.Errorf("no such link in this branch") +} + +// shiftFromPath extracts from a given path (as a slice of strings) +// the given number of elements as a single string, returning whatever +// it has not taken. +// +// Examples: +// ["0", "a", "something"] and 1 -> "0" and ["a", "something"] +// ["ab", "c", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +// ["abc", "d", "1"] and 2 -> "ab" and ["c", "d", "1"] +func shiftFromPath(p []string, i int) (string, []string) { + var ( + out string + rest []string + ) + + for _, pe := range p { + re := "" + for _, c := range pe { + if len(out) < i { + out += string(c) + } else { + re += string(c) + } + } + + if len(out) == i && re != "" { + rest = append(rest, re) + } + } + + return out, rest +} + +// getHexIndex returns to you the integer 0 - 15 equivalent to your +// string character if applicable, or -1 otherwise. +func getHexIndex(s string) int { + if len(s) != 1 { + return -1 + } + + c := byte(s[0]) + switch { + case '0' <= c && c <= '9': + return int(c - '0') + case 'a' <= c && c <= 'f': + return int(c - 'a' + 10) + } + + return -1 +} diff --git a/statediff/indexer/ipfs/models.go b/statediff/indexer/ipfs/models.go new file mode 100644 index 000000000000..eb0312beb585 --- /dev/null +++ b/statediff/indexer/ipfs/models.go @@ -0,0 +1,22 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipfs + +type BlockModel struct { + CID string `db:"key"` + Data []byte `db:"data"` +} diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go new file mode 100644 index 000000000000..7fe49d1f5e56 --- /dev/null +++ b/statediff/indexer/mocks/test_data.go @@ -0,0 +1,455 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package mocks + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ipfs/go-block-format" + "github.com/multiformats/go-multihash" + log "github.com/sirupsen/logrus" + + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/ethereum/go-ethereum/statediff/testhelpers" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" +) + +// Test variables +var ( + // block data + BlockNumber = big.NewInt(1) + MockHeader = types.Header{ + Time: 0, + Number: new(big.Int).Set(BlockNumber), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(5000000), + Extra: []byte{}, + } + MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts() + ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) + MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) + MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) + Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") + AnotherAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") + ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce()) + ContractHash = crypto.Keccak256Hash(ContractAddress.Bytes()).String() + MockContractByteCode = []byte{0, 1, 2, 3, 4, 5} + mockTopic11 = common.HexToHash("0x04") + mockTopic12 = common.HexToHash("0x06") + mockTopic21 = common.HexToHash("0x05") + mockTopic22 = common.HexToHash("0x07") + MockLog1 = &types.Log{ + Address: Address, + Topics: []common.Hash{mockTopic11, mockTopic12}, + Data: []byte{}, + } + MockLog2 = &types.Log{ + Address: AnotherAddress, + Topics: []common.Hash{mockTopic21, mockTopic22}, + Data: []byte{}, + } + HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256) + HeaderMhKey = shared.MultihashKeyFromCID(HeaderCID) + Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256) + Trx1MhKey = shared.MultihashKeyFromCID(Trx1CID) + Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256) + Trx2MhKey = shared.MultihashKeyFromCID(Trx2CID) + Trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256) + Trx3MhKey = shared.MultihashKeyFromCID(Trx3CID) + Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256) + Rct1MhKey = shared.MultihashKeyFromCID(Rct1CID) + Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256) + Rct2MhKey = shared.MultihashKeyFromCID(Rct2CID) + Rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256) + Rct3MhKey = shared.MultihashKeyFromCID(Rct3CID) + State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256) + State1MhKey = shared.MultihashKeyFromCID(State1CID) + State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256) + State2MhKey = shared.MultihashKeyFromCID(State2CID) + StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256) + StorageMhKey = shared.MultihashKeyFromCID(StorageCID) + MockTrxMeta = []models.TxModel{ + { + CID: "", // This is empty until we go to publish to ipfs + MhKey: "", + Src: SenderAddr.Hex(), + Dst: Address.String(), + Index: 0, + TxHash: MockTransactions[0].Hash().String(), + Data: []byte{}, + Deployment: false, + }, + { + CID: "", + MhKey: "", + Src: SenderAddr.Hex(), + Dst: AnotherAddress.String(), + Index: 1, + TxHash: MockTransactions[1].Hash().String(), + Data: []byte{}, + Deployment: false, + }, + { + CID: "", + MhKey: "", + Src: SenderAddr.Hex(), + Dst: "", + Index: 2, + TxHash: MockTransactions[2].Hash().String(), + Data: MockContractByteCode, + Deployment: true, + }, + } + MockTrxMetaPostPublsh = []models.TxModel{ + { + CID: Trx1CID.String(), // This is empty until we go to publish to ipfs + MhKey: Trx1MhKey, + Src: SenderAddr.Hex(), + Dst: Address.String(), + Index: 0, + TxHash: MockTransactions[0].Hash().String(), + Data: []byte{}, + Deployment: false, + }, + { + CID: Trx2CID.String(), + MhKey: Trx2MhKey, + Src: SenderAddr.Hex(), + Dst: AnotherAddress.String(), + Index: 1, + TxHash: MockTransactions[1].Hash().String(), + Data: []byte{}, + Deployment: false, + }, + { + CID: Trx3CID.String(), + MhKey: Trx3MhKey, + Src: SenderAddr.Hex(), + Dst: "", + Index: 2, + TxHash: MockTransactions[2].Hash().String(), + Data: MockContractByteCode, + Deployment: true, + }, + } + MockRctMeta = []models.ReceiptModel{ + { + CID: "", + MhKey: "", + Topic0s: []string{ + mockTopic11.String(), + }, + Topic1s: []string{ + mockTopic12.String(), + }, + Contract: "", + ContractHash: "", + LogContracts: []string{ + Address.String(), + }, + }, + { + CID: "", + MhKey: "", + Topic0s: []string{ + mockTopic21.String(), + }, + Topic1s: []string{ + mockTopic22.String(), + }, + Contract: "", + ContractHash: "", + LogContracts: []string{ + AnotherAddress.String(), + }, + }, + { + CID: "", + MhKey: "", + Contract: ContractAddress.String(), + ContractHash: ContractHash, + LogContracts: []string{}, + }, + } + MockRctMetaPostPublish = []models.ReceiptModel{ + { + CID: Rct1CID.String(), + MhKey: Rct1MhKey, + Topic0s: []string{ + mockTopic11.String(), + }, + Topic1s: []string{ + mockTopic12.String(), + }, + Contract: "", + ContractHash: "", + LogContracts: []string{ + Address.String(), + }, + }, + { + CID: Rct2CID.String(), + MhKey: Rct2MhKey, + Topic0s: []string{ + mockTopic21.String(), + }, + Topic1s: []string{ + mockTopic22.String(), + }, + Contract: "", + ContractHash: "", + LogContracts: []string{ + AnotherAddress.String(), + }, + }, + { + CID: Rct3CID.String(), + MhKey: Rct3MhKey, + Contract: ContractAddress.String(), + ContractHash: ContractHash, + LogContracts: []string{}, + }, + } + + // statediff data + storageLocation = common.HexToHash("0") + StorageLeafKey = crypto.Keccak256Hash(storageLocation[:]).Bytes() + StorageValue = common.Hex2Bytes("01") + StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") + StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + StoragePartialPath, + StorageValue, + }) + + nonce1 = uint64(1) + ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0" + ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea") + contractPath = common.Bytes2Hex([]byte{'\x06'}) + ContractLeafKey = testhelpers.AddressToLeafKey(ContractAddress) + ContractAccount, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce1, + Balance: big.NewInt(0), + CodeHash: ContractCodeHash.Bytes(), + Root: common.HexToHash(ContractRoot), + }) + ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") + ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + ContractPartialPath, + ContractAccount, + }) + + nonce0 = uint64(0) + AccountRoot = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" + AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + accountPath = common.Bytes2Hex([]byte{'\x0c'}) + AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e") + AccountLeafKey = testhelpers.Account2LeafKey + Account, _ = rlp.EncodeToBytes(state.Account{ + Nonce: nonce0, + Balance: big.NewInt(1000), + CodeHash: AccountCodeHash.Bytes(), + Root: common.HexToHash(AccountRoot), + }) + AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45") + AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{ + AccountPartialPath, + Account, + }) + + StateDiffs = []sdtypes.StateNode{ + { + Path: []byte{'\x06'}, + NodeType: sdtypes.Leaf, + LeafKey: ContractLeafKey, + NodeValue: ContractLeafNode, + StorageNodes: []sdtypes.StorageNode{ + { + Path: []byte{}, + NodeType: sdtypes.Leaf, + LeafKey: StorageLeafKey, + NodeValue: StorageLeafNode, + }, + }, + }, + { + Path: []byte{'\x0c'}, + NodeType: sdtypes.Leaf, + LeafKey: AccountLeafKey, + NodeValue: AccountLeafNode, + StorageNodes: []sdtypes.StorageNode{}, + }, + } + + MockStateNodes = []shared.TrieNode{ + { + LeafKey: common.BytesToHash(ContractLeafKey), + Path: []byte{'\x06'}, + Value: ContractLeafNode, + Type: sdtypes.Leaf, + }, + { + LeafKey: common.BytesToHash(AccountLeafKey), + Path: []byte{'\x0c'}, + Value: AccountLeafNode, + Type: sdtypes.Leaf, + }, + } + MockStateMetaPostPublish = []models.StateNodeModel{ + { + CID: State1CID.String(), + MhKey: State1MhKey, + Path: []byte{'\x06'}, + NodeType: 2, + StateKey: common.BytesToHash(ContractLeafKey).Hex(), + }, + { + CID: State2CID.String(), + MhKey: State2MhKey, + Path: []byte{'\x0c'}, + NodeType: 2, + StateKey: common.BytesToHash(AccountLeafKey).Hex(), + }, + } + MockStorageNodes = map[string][]shared.TrieNode{ + contractPath: { + { + LeafKey: common.BytesToHash(StorageLeafKey), + Value: StorageLeafNode, + Type: sdtypes.Leaf, + Path: []byte{}, + }, + }, + } + + // aggregate payloads + MockCIDPayload = shared.CIDPayload{ + HeaderCID: models.HeaderModel{ + BlockHash: MockBlock.Hash().String(), + BlockNumber: MockBlock.Number().String(), + CID: HeaderCID.String(), + MhKey: HeaderMhKey, + ParentHash: MockBlock.ParentHash().String(), + TotalDifficulty: MockBlock.Difficulty().String(), + Reward: "5000000000000000000", + StateRoot: MockBlock.Root().String(), + RctRoot: MockBlock.ReceiptHash().String(), + TxRoot: MockBlock.TxHash().String(), + UncleRoot: MockBlock.UncleHash().String(), + Bloom: MockBlock.Bloom().Bytes(), + Timestamp: MockBlock.Time(), + }, + UncleCIDs: []models.UncleModel{}, + TransactionCIDs: MockTrxMetaPostPublsh, + ReceiptCIDs: map[common.Hash]models.ReceiptModel{ + MockTransactions[0].Hash(): MockRctMetaPostPublish[0], + MockTransactions[1].Hash(): MockRctMetaPostPublish[1], + MockTransactions[2].Hash(): MockRctMetaPostPublish[2], + }, + StateNodeCIDs: MockStateMetaPostPublish, + StorageNodeCIDs: map[string][]models.StorageNodeModel{ + contractPath: { + { + CID: StorageCID.String(), + MhKey: StorageMhKey, + Path: []byte{}, + StorageKey: common.BytesToHash(StorageLeafKey).Hex(), + NodeType: 2, + }, + }, + }, + StateAccounts: map[string]models.StateAccountModel{ + contractPath: { + Balance: big.NewInt(0).String(), + Nonce: nonce1, + CodeHash: ContractCodeHash.Bytes(), + StorageRoot: common.HexToHash(ContractRoot).String(), + }, + accountPath: { + Balance: big.NewInt(1000).String(), + Nonce: nonce0, + CodeHash: AccountCodeHash.Bytes(), + StorageRoot: common.HexToHash(AccountRoot).String(), + }, + }, + } + + HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID) + Trx1IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID) + Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID) + Trx3IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(2), Trx3CID) + Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID) + Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID) + Rct3IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(2), Rct3CID) + State1IPLD, _ = blocks.NewBlockWithCid(ContractLeafNode, State1CID) + State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID) + StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID) +) + +// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs +func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) { + // make transactions + trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) + trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) + trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) + transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber)) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Fatal(err) + } + signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey) + if err != nil { + log.Fatal(err) + } + signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey) + if err != nil { + log.Fatal(err) + } + signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey) + if err != nil { + log.Fatal(err) + } + SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx + if err != nil { + log.Fatal(err) + } + // make receipts + mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50) + mockReceipt1.Logs = []*types.Log{MockLog1} + mockReceipt1.TxHash = signedTrx1.Hash() + mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt2.Logs = []*types.Log{MockLog2} + mockReceipt2.TxHash = signedTrx2.Hash() + mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75) + mockReceipt3.Logs = []*types.Log{} + mockReceipt3.TxHash = signedTrx3.Hash() + return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr +} diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go new file mode 100644 index 000000000000..8e8ebf6244bb --- /dev/null +++ b/statediff/indexer/models/models.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package models + +import "github.com/lib/pq" + +// HeaderModel is the db model for eth.header_cids +type HeaderModel struct { + ID int64 `db:"id"` + BlockNumber string `db:"block_number"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + TotalDifficulty string `db:"td"` + NodeID int64 `db:"node_id"` + Reward string `db:"reward"` + StateRoot string `db:"state_root"` + UncleRoot string `db:"uncle_root"` + TxRoot string `db:"tx_root"` + RctRoot string `db:"receipt_root"` + Bloom []byte `db:"bloom"` + Timestamp uint64 `db:"timestamp"` + TimesValidated int64 `db:"times_validated"` +} + +// UncleModel is the db model for eth.uncle_cids +type UncleModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + BlockHash string `db:"block_hash"` + ParentHash string `db:"parent_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Reward string `db:"reward"` +} + +// TxModel is the db model for eth.transaction_cids +type TxModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + Index int64 `db:"index"` + TxHash string `db:"tx_hash"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Dst string `db:"dst"` + Src string `db:"src"` + Data []byte `db:"tx_data"` + Deployment bool `db:"deployment"` +} + +// ReceiptModel is the db model for eth.receipt_cids +type ReceiptModel struct { + ID int64 `db:"id"` + TxID int64 `db:"tx_id"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Contract string `db:"contract"` + ContractHash string `db:"contract_hash"` + LogContracts pq.StringArray `db:"log_contracts"` + Topic0s pq.StringArray `db:"topic0s"` + Topic1s pq.StringArray `db:"topic1s"` + Topic2s pq.StringArray `db:"topic2s"` + Topic3s pq.StringArray `db:"topic3s"` +} + +// StateNodeModel is the db model for eth.state_cids +type StateNodeModel struct { + ID int64 `db:"id"` + HeaderID int64 `db:"header_id"` + Path []byte `db:"state_path"` + StateKey string `db:"state_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeModel is the db model for eth.storage_cids +type StorageNodeModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key +type StorageNodeWithStateKeyModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Path []byte `db:"storage_path"` + StateKey string `db:"state_leaf_key"` + StorageKey string `db:"storage_leaf_key"` + NodeType int `db:"node_type"` + CID string `db:"cid"` + MhKey string `db:"mh_key"` + Diff bool `db:"diff"` +} + +// StateAccountModel is a db model for an eth state account (decoded value of state leaf node) +type StateAccountModel struct { + ID int64 `db:"id"` + StateID int64 `db:"state_id"` + Balance string `db:"balance"` + Nonce uint64 `db:"nonce"` + CodeHash []byte `db:"code_hash"` + StorageRoot string `db:"storage_root"` +} diff --git a/statediff/indexer/node/node.go b/statediff/indexer/node/node.go new file mode 100644 index 000000000000..527546efa277 --- /dev/null +++ b/statediff/indexer/node/node.go @@ -0,0 +1,25 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package node + +type Info struct { + GenesisBlock string + NetworkID string + ChainID uint64 + ID string + ClientName string +} diff --git a/statediff/indexer/postgres/config.go b/statediff/indexer/postgres/config.go new file mode 100644 index 000000000000..c2de0a6bfcc9 --- /dev/null +++ b/statediff/indexer/postgres/config.go @@ -0,0 +1,59 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" +) + +// Env variables +const ( + DATABASE_NAME = "DATABASE_NAME" + DATABASE_HOSTNAME = "DATABASE_HOSTNAME" + DATABASE_PORT = "DATABASE_PORT" + DATABASE_USER = "DATABASE_USER" + DATABASE_PASSWORD = "DATABASE_PASSWORD" + DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS" + DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS" + DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME" +) + +type ConnectionParams struct { + Hostname string + Name string + User string + Password string + Port int +} + +type ConnectionConfig struct { + MaxIdle int + MaxOpen int + MaxLifetime int +} + +func DbConnectionString(params ConnectionParams) string { + if len(params.User) > 0 && len(params.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + params.User, params.Password, params.Hostname, params.Port, params.Name) + } + if len(params.User) > 0 && len(params.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + params.User, params.Hostname, params.Port, params.Name) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", params.Hostname, params.Port, params.Name) +} diff --git a/statediff/indexer/postgres/config_test.go b/statediff/indexer/postgres/config_test.go new file mode 100644 index 000000000000..e93a4892d1e4 --- /dev/null +++ b/statediff/indexer/postgres/config_test.go @@ -0,0 +1,48 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "bytes" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/spf13/viper" +) + +var vulcanizeConfig = []byte(` +[database] +name = "dbname" +hostname = "localhost" +port = 5432 +`) + +var _ = Describe("Loading the config", func() { + It("reads the private config using the environment", func() { + viper.SetConfigName("config") + // viper.AddConfigPath("$GOPATH/src/github.com/ethereum/go-ethereum/statediff/environments/") + + testConfig := viper.New() + testConfig.SetConfigType("toml") + err := testConfig.ReadConfig(bytes.NewBuffer(vulcanizeConfig)) + Expect(err).To(BeNil()) + Expect(testConfig.Get("database.hostname")).To(Equal("localhost")) + Expect(testConfig.Get("database.name")).To(Equal("dbname")) + Expect(testConfig.Get("database.port")).To(Equal(int64(5432))) + }) + +}) diff --git a/statediff/indexer/postgres/errors.go b/statediff/indexer/postgres/errors.go new file mode 100644 index 000000000000..f368f900b2e5 --- /dev/null +++ b/statediff/indexer/postgres/errors.go @@ -0,0 +1,53 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" +) + +const ( + BeginTransactionFailedMsg = "failed to begin transaction" + DbConnectionFailedMsg = "db connection failed" + DeleteQueryFailedMsg = "delete query failed" + InsertQueryFailedMsg = "insert query failed" + SettingNodeFailedMsg = "unable to set db node" +) + +func ErrBeginTransactionFailed(beginErr error) error { + return formatError(BeginTransactionFailedMsg, beginErr.Error()) +} + +func ErrDBConnectionFailed(connectErr error) error { + return formatError(DbConnectionFailedMsg, connectErr.Error()) +} + +func ErrDBDeleteFailed(deleteErr error) error { + return formatError(DeleteQueryFailedMsg, deleteErr.Error()) +} + +func ErrDBInsertFailed(insertErr error) error { + return formatError(InsertQueryFailedMsg, insertErr.Error()) +} + +func ErrUnableToSetNode(setErr error) error { + return formatError(SettingNodeFailedMsg, setErr.Error()) +} + +func formatError(msg, err string) error { + return fmt.Errorf("%s: %s", msg, err) +} diff --git a/statediff/indexer/postgres/postgres.go b/statediff/indexer/postgres/postgres.go new file mode 100644 index 000000000000..455dac306778 --- /dev/null +++ b/statediff/indexer/postgres/postgres.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "time" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" //postgres driver + + "github.com/ethereum/go-ethereum/statediff/indexer/node" +) + +type DB struct { + *sqlx.DB + Node node.Info + NodeID int64 +} + +func NewDB(connectString string, config ConnectionConfig, node node.Info) (*DB, error) { + db, connectErr := sqlx.Connect("postgres", connectString) + if connectErr != nil { + return &DB{}, ErrDBConnectionFailed(connectErr) + } + if config.MaxOpen > 0 { + db.SetMaxOpenConns(config.MaxOpen) + } + if config.MaxIdle > 0 { + db.SetMaxIdleConns(config.MaxIdle) + } + if config.MaxLifetime > 0 { + lifetime := time.Duration(config.MaxLifetime) * time.Second + db.SetConnMaxLifetime(lifetime) + } + pg := DB{DB: db, Node: node} + nodeErr := pg.CreateNode(&node) + if nodeErr != nil { + return &DB{}, ErrUnableToSetNode(nodeErr) + } + return &pg, nil +} + +func (db *DB) CreateNode(node *node.Info) error { + var nodeID int64 + err := db.QueryRow( + `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (genesis_block, network_id, node_id, chain_id) + DO UPDATE + SET genesis_block = $1, + network_id = $2, + node_id = $3, + client_name = $4, + chain_id = $5 + RETURNING id`, + node.GenesisBlock, node.NetworkID, node.ID, node.ClientName, node.ChainID).Scan(&nodeID) + if err != nil { + return ErrUnableToSetNode(err) + } + db.NodeID = nodeID + return nil +} diff --git a/statediff/indexer/postgres/postgres_suite_test.go b/statediff/indexer/postgres/postgres_suite_test.go new file mode 100644 index 000000000000..8a991b39cf18 --- /dev/null +++ b/statediff/indexer/postgres/postgres_suite_test.go @@ -0,0 +1,36 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "io/ioutil" + "testing" + + log "github.com/sirupsen/logrus" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func init() { + log.SetOutput(ioutil.Discard) +} + +func TestPostgres(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Postgres Suite") +} diff --git a/statediff/indexer/postgres/postgres_test.go b/statediff/indexer/postgres/postgres_test.go new file mode 100644 index 000000000000..fd4e0146459e --- /dev/null +++ b/statediff/indexer/postgres/postgres_test.go @@ -0,0 +1,104 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres_test + +import ( + "fmt" + "strings" + + "math/big" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/statediff/node" + "github.com/ethereum/go-ethereum/statediff/postgres" +) + +var DBConfig postgres.Config + +var _ = Describe("Postgres DB", func() { + var sqlxdb *sqlx.DB + + It("connects to the database", func() { + var err error + pgConfig := postgres.DbConnectionString(DBConfig) + + sqlxdb, err = sqlx.Connect("postgres", pgConfig) + + Expect(err).Should(BeNil()) + Expect(sqlxdb).ShouldNot(BeNil()) + }) + + It("serializes big.Int to db", func() { + // postgres driver doesn't support go big.Int type + // various casts in golang uint64, int64, overflow for + // transaction value (in wei) even though + // postgres numeric can handle an arbitrary + // sized int, so use string representation of big.Int + // and cast on insert + + pgConnectString := postgres.DbConnectionString(DBConfig) + db, err := sqlx.Connect("postgres", pgConnectString) + Expect(err).NotTo(HaveOccurred()) + + bi := new(big.Int) + bi.SetString("34940183920000000000", 10) + Expect(bi.String()).To(Equal("34940183920000000000")) + + defer db.Exec(`DROP TABLE IF EXISTS example`) + _, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )") + Expect(err).ToNot(HaveOccurred()) + + sqlStatement := ` + INSERT INTO example (id, data) + VALUES (1, cast($1 AS NUMERIC))` + _, err = db.Exec(sqlStatement, bi.String()) + Expect(err).ToNot(HaveOccurred()) + + var data string + err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data) + Expect(err).ToNot(HaveOccurred()) + + Expect(bi.String()).To(Equal(data)) + actual := new(big.Int) + actual.SetString(data, 10) + Expect(actual).To(Equal(bi)) + }) + + It("throws error when can't connect to the database", func() { + invalidDatabase := postgres.Config{} + node := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewDB(invalidDatabase, node) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(postgres.DbConnectionFailedMsg)) + }) + + It("throws error when can't create node", func() { + badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100)) + node := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"} + + _, err := postgres.NewDB(DBConfig, node) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(postgres.SettingNodeFailedMsg)) + }) +}) diff --git a/statediff/indexer/reward.go b/statediff/indexer/reward.go new file mode 100644 index 000000000000..47e3f17b9151 --- /dev/null +++ b/statediff/indexer/reward.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" +) + +func CalcEthBlockReward(header *types.Header, uncles []*types.Header, txs types.Transactions, receipts types.Receipts) *big.Int { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + transactionFees := calcEthTransactionFees(txs, receipts) + uncleInclusionRewards := calcEthUncleInclusionRewards(header, uncles) + tmp := transactionFees.Add(transactionFees, uncleInclusionRewards) + return tmp.Add(tmp, staticBlockReward) +} + +func CalcUncleMinerReward(blockNumber, uncleBlockNumber uint64) *big.Int { + staticBlockReward := staticRewardByBlockNumber(blockNumber) + rewardDiv8 := staticBlockReward.Div(staticBlockReward, big.NewInt(8)) + mainBlock := new(big.Int).SetUint64(blockNumber) + uncleBlock := new(big.Int).SetUint64(uncleBlockNumber) + uncleBlockPlus8 := uncleBlock.Add(uncleBlock, big.NewInt(8)) + uncleBlockPlus8MinusMainBlock := uncleBlockPlus8.Sub(uncleBlockPlus8, mainBlock) + return rewardDiv8.Mul(rewardDiv8, uncleBlockPlus8MinusMainBlock) +} + +func staticRewardByBlockNumber(blockNumber uint64) *big.Int { + staticBlockReward := new(big.Int) + //https://blog.ethereum.org/2017/10/12/byzantium-hf-announcement/ + if blockNumber >= 7280000 { + staticBlockReward.SetString("2000000000000000000", 10) + } else if blockNumber >= 4370000 { + staticBlockReward.SetString("3000000000000000000", 10) + } else { + staticBlockReward.SetString("5000000000000000000", 10) + } + return staticBlockReward +} + +func calcEthTransactionFees(txs types.Transactions, receipts types.Receipts) *big.Int { + transactionFees := new(big.Int) + for i, transaction := range txs { + receipt := receipts[i] + gasPrice := big.NewInt(transaction.GasPrice().Int64()) + gasUsed := big.NewInt(int64(receipt.GasUsed)) + transactionFee := gasPrice.Mul(gasPrice, gasUsed) + transactionFees = transactionFees.Add(transactionFees, transactionFee) + } + return transactionFees +} + +func calcEthUncleInclusionRewards(header *types.Header, uncles []*types.Header) *big.Int { + uncleInclusionRewards := new(big.Int) + for range uncles { + staticBlockReward := staticRewardByBlockNumber(header.Number.Uint64()) + staticBlockReward.Div(staticBlockReward, big.NewInt(32)) + uncleInclusionRewards.Add(uncleInclusionRewards, staticBlockReward) + } + return uncleInclusionRewards +} diff --git a/statediff/indexer/shared/chain_type.go b/statediff/indexer/shared/chain_type.go new file mode 100644 index 000000000000..c3dedfe38e75 --- /dev/null +++ b/statediff/indexer/shared/chain_type.go @@ -0,0 +1,78 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "errors" + "strings" +) + +// ChainType enum for specifying blockchain +type ChainType int + +const ( + UnknownChain ChainType = iota + Ethereum + Bitcoin + Omni + EthereumClassic +) + +func (c ChainType) String() string { + switch c { + case Ethereum: + return "Ethereum" + case Bitcoin: + return "Bitcoin" + case Omni: + return "Omni" + case EthereumClassic: + return "EthereumClassic" + default: + return "" + } +} + +func (c ChainType) API() string { + switch c { + case Ethereum: + return "eth" + case Bitcoin: + return "btc" + case Omni: + return "omni" + case EthereumClassic: + return "etc" + default: + return "" + } +} + +func NewChainType(name string) (ChainType, error) { + switch strings.ToLower(name) { + case "ethereum", "eth": + return Ethereum, nil + case "bitcoin", "btc", "xbt": + return Bitcoin, nil + case "omni": + return Omni, nil + case "classic", "etc": + return EthereumClassic, nil + default: + return UnknownChain, errors.New("invalid name for chain") + } +} diff --git a/statediff/indexer/shared/constants.go b/statediff/indexer/shared/constants.go new file mode 100644 index 000000000000..3dc2994c4216 --- /dev/null +++ b/statediff/indexer/shared/constants.go @@ -0,0 +1,22 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +const ( + DefaultMaxBatchSize uint64 = 100 + DefaultMaxBatchNumber int64 = 50 +) diff --git a/statediff/indexer/shared/data_type.go b/statediff/indexer/shared/data_type.go new file mode 100644 index 000000000000..01fed57f76af --- /dev/null +++ b/statediff/indexer/shared/data_type.go @@ -0,0 +1,101 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "fmt" + "strings" +) + +// DataType is an enum to loosely represent type of chain data +type DataType int + +const ( + UnknownDataType DataType = iota - 1 + Full + Headers + Uncles + Transactions + Receipts + State + Storage +) + +// String() method to resolve ReSyncType enum +func (r DataType) String() string { + switch r { + case Full: + return "full" + case Headers: + return "headers" + case Uncles: + return "uncles" + case Transactions: + return "transactions" + case Receipts: + return "receipts" + case State: + return "state" + case Storage: + return "storage" + default: + return "unknown" + } +} + +// GenerateDataTypeFromString +func GenerateDataTypeFromString(str string) (DataType, error) { + switch strings.ToLower(str) { + case "full", "f": + return Full, nil + case "headers", "header", "h": + return Headers, nil + case "uncles", "u": + return Uncles, nil + case "transactions", "transaction", "trxs", "txs", "trx", "tx", "t": + return Transactions, nil + case "receipts", "receipt", "rcts", "rct", "r": + return Receipts, nil + case "state": + return State, nil + case "storage": + return Storage, nil + default: + return UnknownDataType, fmt.Errorf("unrecognized resync type: %s", str) + } +} + +func SupportedDataType(d DataType) (bool, error) { + switch d { + case Full: + return true, nil + case Headers: + return true, nil + case Uncles: + return true, nil + case Transactions: + return true, nil + case Receipts: + return true, nil + case State: + return true, nil + case Storage: + return true, nil + default: + return true, nil + } +} diff --git a/statediff/indexer/shared/functions.go b/statediff/indexer/shared/functions.go new file mode 100644 index 000000000000..26391decb0b5 --- /dev/null +++ b/statediff/indexer/shared/functions.go @@ -0,0 +1,107 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-ds-help" + node "github.com/ipfs/go-ipld-format" + "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" +) + +// HandleZeroAddrPointer will return an emtpy string for a nil address pointer +func HandleZeroAddrPointer(to *common.Address) string { + if to == nil { + return "" + } + return to.Hex() +} + +// HandleZeroAddr will return an empty string for a 0 value address +func HandleZeroAddr(to common.Address) string { + if to.Hex() == "0x0000000000000000000000000000000000000000" { + return "" + } + return to.Hex() +} + +// Rollback sql transaction and log any error +func Rollback(tx *sqlx.Tx) { + if err := tx.Rollback(); err != nil { + logrus.Error(err) + } +} + +// PublishIPLD is used to insert an IPLD into Postgres blockstore with the provided tx +func PublishIPLD(tx *sqlx.Tx, i node.Node) error { + dbKey := dshelp.MultihashToDsKey(i.Cid().Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + raw := i.RawData() + _, err := tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) + return err +} + +// FetchIPLD is used to retrieve an ipld from Postgres blockstore with the provided tx and cid string +func FetchIPLD(tx *sqlx.Tx, cid string) ([]byte, error) { + mhKey, err := MultihashKeyFromCIDString(cid) + if err != nil { + return nil, err + } + pgStr := `SELECT data FROM public.blocks WHERE key = $1` + var block []byte + return block, tx.Get(&block, pgStr, mhKey) +} + +// FetchIPLDByMhKey is used to retrieve an ipld from Postgres blockstore with the provided tx and mhkey string +func FetchIPLDByMhKey(tx *sqlx.Tx, mhKey string) ([]byte, error) { + pgStr := `SELECT data FROM public.blocks WHERE key = $1` + var block []byte + return block, tx.Get(&block, pgStr, mhKey) +} + +// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string +func MultihashKeyFromCID(c cid.Cid) string { + dbKey := dshelp.MultihashToDsKey(c.Hash()) + return blockstore.BlockPrefix.String() + dbKey.String() +} + +// MultihashKeyFromCIDString converts a cid string into a blockstore-prefixed multihash db key string +func MultihashKeyFromCIDString(c string) (string, error) { + dc, err := cid.Decode(c) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(dc.Hash()) + return blockstore.BlockPrefix.String() + dbKey.String(), nil +} + +// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx +func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) { + c, err := ipld.RawdataToCid(codec, raw, mh) + if err != nil { + return "", err + } + dbKey := dshelp.MultihashToDsKey(c.Hash()) + prefixedKey := blockstore.BlockPrefix.String() + dbKey.String() + _, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw) + return c.String(), err +} diff --git a/statediff/indexer/shared/test_helpers.go b/statediff/indexer/shared/test_helpers.go new file mode 100644 index 000000000000..e83bbc59aa80 --- /dev/null +++ b/statediff/indexer/shared/test_helpers.go @@ -0,0 +1,65 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import ( + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" +) + +// SetupDB is use to setup a db for watcher tests +func SetupDB() (*postgres.DB, error) { + uri := postgres.DbConnectionString(postgres.ConnectionParams{ + User: "vulcanize", + Password: "libertad", + Hostname: "localhost", + Name: "vulcanize_testing", + Port: 5432, + }) + return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{}) +} + +// ListContainsString used to check if a list of strings contains a particular string +func ListContainsString(sss []string, s string) bool { + for _, str := range sss { + if s == str { + return true + } + } + return false +} + +// TestCID creates a basic CID for testing purposes +func TestCID(b []byte) cid.Cid { + pref := cid.Prefix{ + Version: 1, + Codec: cid.Raw, + MhType: multihash.KECCAK_256, + MhLength: -1, + } + c, _ := pref.Sum(b) + return c +} + +// PublishMockIPLD writes a mhkey-data pair to the public.blocks table so that test data can FK reference the mhkey +func PublishMockIPLD(db *postgres.DB, mhKey string, mockData []byte) error { + _, err := db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, mhKey, mockData) + return err +} diff --git a/statediff/indexer/shared/types.go b/statediff/indexer/shared/types.go new file mode 100644 index 000000000000..a9792522bd00 --- /dev/null +++ b/statediff/indexer/shared/types.go @@ -0,0 +1,42 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package shared + +import "github.com/ethereum/go-ethereum/common" +import "github.com/ethereum/go-ethereum/statediff/types" +import "github.com/ethereum/go-ethereum/statediff/indexer/models" + +// Trie struct used to flag node as leaf or not +type TrieNode struct { + Path []byte + LeafKey common.Hash + Value []byte + Type types.NodeType +} + +// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres +// Returned by IPLDPublisher +// Passed to CIDIndexer +type CIDPayload struct { + HeaderCID models.HeaderModel + UncleCIDs []models.UncleModel + TransactionCIDs []models.TxModel + ReceiptCIDs map[common.Hash]models.ReceiptModel + StateNodeCIDs []models.StateNodeModel + StateAccounts map[string]models.StateAccountModel + StorageNodeCIDs map[string][]models.StorageNodeModel +} diff --git a/statediff/indexer/test_helpers.go b/statediff/indexer/test_helpers.go new file mode 100644 index 000000000000..7a93e8e35cdc --- /dev/null +++ b/statediff/indexer/test_helpers.go @@ -0,0 +1,66 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + . "github.com/onsi/gomega" + + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" +) + +// TearDownDB is used to tear down the watcher dbs after tests +func TearDownDB(db *postgres.DB) { + tx, err := db.Beginx() + Expect(err).NotTo(HaveOccurred()) + + _, err = tx.Exec(`DELETE FROM eth.header_cids`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.transaction_cids`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.receipt_cids`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.state_cids`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM eth.storage_cids`) + Expect(err).NotTo(HaveOccurred()) + _, err = tx.Exec(`DELETE FROM blocks`) + Expect(err).NotTo(HaveOccurred()) + + err = tx.Commit() + Expect(err).NotTo(HaveOccurred()) +} + +// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string +func TxModelsContainsCID(txs []models.TxModel, cid string) bool { + for _, tx := range txs { + if tx.CID == cid { + return true + } + } + return false +} + +// ListContainsBytes used to check if a list of byte arrays contains a particular byte array +func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool { + for _, rct := range rcts { + if rct.CID == cid { + return true + } + } + return false +} diff --git a/statediff/indexer/utils/utils.go b/statediff/indexer/utils/utils.go new file mode 100644 index 000000000000..9404c4fb6643 --- /dev/null +++ b/statediff/indexer/utils/utils.go @@ -0,0 +1,34 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package utils + +import ( + "errors" + + "github.com/sirupsen/logrus" + + "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" +) + +func LoadPostgres(database postgres.Config, node node.Info) postgres.DB { + db, err := postgres.NewDB(database, node) + if err != nil { + logrus.Fatal("Error loading postgres: ", err) + } + return *db +} diff --git a/statediff/indexer/writer.go b/statediff/indexer/writer.go new file mode 100644 index 000000000000..de92d036abb5 --- /dev/null +++ b/statediff/indexer/writer.go @@ -0,0 +1,159 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/jmoiron/sqlx" + + "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" +) + +var ( + nullHash = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000") +) + +// Indexer satisfies the Indexer interface for ethereum +type PostgresCIDWriter struct { + db *postgres.DB +} + +// NewPostgresCIDWriter creates a new pointer to a Indexer which satisfies the PostgresCIDWriter interface +func NewPostgresCIDWriter(db *postgres.DB) *PostgresCIDWriter { + return &PostgresCIDWriter{ + db: db, + } +} + +func (in *PostgresCIDWriter) indexHeaderCID(tx *sqlx.Tx, header models.HeaderModel) (int64, error) { + var headerID int64 + err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1) + RETURNING id`, + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot, + header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1).Scan(&headerID) + return headerID, err +} + +func (in *PostgresCIDWriter) indexUncleCID(tx *sqlx.Tx, uncle models.UncleModel, headerID int64) error { + _, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward, mh_key) = ($3, $4, $5, $6)`, + uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey) + return err +} + +func (in *PostgresCIDWriter) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload shared.CIDPayload, headerID int64) error { + for _, trxCidMeta := range payload.TransactionCIDs { + var txID int64 + err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, deployment) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data, deployment) = ($3, $4, $5, $6, $7, $8, $9) + RETURNING id`, + headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src, trxCidMeta.Index, trxCidMeta.MhKey, trxCidMeta.Data, trxCidMeta.Deployment).Scan(&txID) + if err != nil { + return err + } + receiptCidMeta, ok := payload.ReceiptCIDs[common.HexToHash(trxCidMeta.TxHash)] + if ok { + if err := in.indexReceiptCID(tx, receiptCidMeta, txID); err != nil { + return err + } + } + } + return nil +} + +func (in *PostgresCIDWriter) indexTransactionCID(tx *sqlx.Tx, transaction models.TxModel, headerID int64) (int64, error) { + var txID int64 + err := tx.QueryRowx(`INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, deployment) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src, index, mh_key, tx_data, deployment) = ($3, $4, $5, $6, $7, $8, $9) + RETURNING id`, + headerID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Deployment).Scan(&txID) + return txID, err +} + +func (in *PostgresCIDWriter) indexReceiptCID(tx *sqlx.Tx, rct models.ReceiptModel, txID int64) error { + _, err := tx.Exec(`INSERT INTO eth.receipt_cids (tx_id, cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (tx_id) DO UPDATE SET (cid, contract, contract_hash, topic0s, topic1s, topic2s, topic3s, log_contracts, mh_key) = ($2, $3, $4, $5, $6, $7, $8, $9, $10)`, + txID, rct.CID, rct.Contract, rct.ContractHash, rct.Topic0s, rct.Topic1s, rct.Topic2s, rct.Topic3s, rct.LogContracts, rct.MhKey) + return err +} + +func (in *PostgresCIDWriter) indexStateAndStorageCIDs(tx *sqlx.Tx, payload shared.CIDPayload, headerID int64) error { + for _, stateCID := range payload.StateNodeCIDs { + var stateID int64 + var stateKey string + if stateCID.StateKey != nullHash.String() { + stateKey = stateCID.StateKey + } + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) + RETURNING id`, + headerID, stateKey, stateCID.CID, stateCID.Path, stateCID.NodeType, true, stateCID.MhKey).Scan(&stateID) + if err != nil { + return err + } + // If we have a state leaf node, index the associated account and storage nodes + if stateCID.NodeType == 2 { + statePath := common.Bytes2Hex(stateCID.Path) + for _, storageCID := range payload.StorageNodeCIDs[statePath] { + if err := in.indexStorageCID(tx, storageCID, stateID); err != nil { + return err + } + } + if stateAccount, ok := payload.StateAccounts[statePath]; ok { + if err := in.indexStateAccount(tx, stateAccount, stateID); err != nil { + return err + } + } + } + } + return nil +} + +func (in *PostgresCIDWriter) indexStateCID(tx *sqlx.Tx, stateNode models.StateNodeModel, headerID int64) (int64, error) { + var stateID int64 + var stateKey string + if stateNode.StateKey != nullHash.String() { + stateKey = stateNode.StateKey + } + err := tx.QueryRowx(`INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) + RETURNING id`, + headerID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey).Scan(&stateID) + return stateID, err +} + +func (in *PostgresCIDWriter) indexStateAccount(tx *sqlx.Tx, stateAccount models.StateAccountModel, stateID int64) error { + _, err := tx.Exec(`INSERT INTO eth.state_accounts (state_id, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (state_id) DO UPDATE SET (balance, nonce, code_hash, storage_root) = ($2, $3, $4, $5)`, + stateID, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot) + return err +} + +func (in *PostgresCIDWriter) indexStorageCID(tx *sqlx.Tx, storageCID models.StorageNodeModel, stateID int64) error { + var storageKey string + if storageCID.StorageKey != nullHash.String() { + storageKey = storageCID.StorageKey + } + _, err := tx.Exec(`INSERT INTO eth.storage_cids (state_id, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (state_id, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)`, + stateID, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, true, storageCID.MhKey) + return err +} diff --git a/statediff/service.go b/statediff/service.go index aec5a34bad03..e65f2780773e 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "math/big" + "strconv" "sync" "sync/atomic" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -35,6 +37,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/trie" + + ind "github.com/ethereum/go-ethereum/statediff/indexer" + nodeinfo "github.com/ethereum/go-ethereum/statediff/indexer/node" + "github.com/ethereum/go-ethereum/statediff/indexer/postgres" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) const chainEventChanSize = 20000 @@ -65,6 +72,8 @@ type IService interface { StateTrieAt(blockNumber uint64, params Params) (*Payload, error) // Method to stream out all code and codehash pairs StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- CodeAndCodeHash, quitChan chan<- bool) + // Method to write state diff object directly to DB + WriteStateDiffAt(blockNumber uint64, params Params) error } // Service is the underlying struct for the state diffing service @@ -85,10 +94,32 @@ type Service struct { lastBlock *types.Block // Whether or not we have any subscribers; only if we do, do we processes state diffs subscribers int32 + // Interface for publishing statediffs as PG-IPLD objects + indexer ind.Indexer } // NewStateDiffService creates a new statediff.Service -func NewStateDiffService(blockChain *core.BlockChain) (*Service, error) { +func NewStateDiffService(ethServ *eth.Ethereum, dbParams *[3]string) (*Service, error) { + blockChain := ethServ.BlockChain() + var indexer ind.Indexer + if dbParams != nil { + info := nodeinfo.Info{ + GenesisBlock: blockChain.Genesis().Hash().Hex(), + NetworkID: strconv.FormatUint(ethServ.NetVersion(), 10), + // ChainID: blockChain.Config().ChainID.String(), + ChainID: blockChain.Config().ChainID.Uint64(), + ID: dbParams[1], + ClientName: dbParams[2], + } + + // TODO: pass max idle, open, lifetime? + db, err := postgres.NewDB(dbParams[0], postgres.ConnectionConfig{}, info) + if err != nil { + return nil, err + } + indexer = ind.NewStateDiffIndexer(blockChain.Config(), db) + } + return &Service{ Mutex: sync.Mutex{}, BlockChain: blockChain, @@ -96,6 +127,7 @@ func NewStateDiffService(blockChain *core.BlockChain) (*Service, error) { QuitChan: make(chan bool), Subscriptions: make(map[common.Hash]map[rpc.ID]Subscription), SubscriptionTypes: make(map[common.Hash]Params), + indexer: indexer, }, nil } @@ -405,3 +437,47 @@ func (sds *Service) StreamCodeAndCodeHash(blockNumber uint64, outChan chan<- Cod } }() } + +// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database +// This operation cannot be performed back past the point of db pruning; it requires an archival node +// for historical data +func (sds *Service) WriteStateDiffAt(blockNumber uint64, params Params) error { + log.Info(fmt.Sprintf("writing state diff at block %d", blockNumber)) + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + parentRoot := common.Hash{} + if blockNumber != 0 { + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + parentRoot = parentBlock.Root() + } + return sds.writeStateDiff(currentBlock, parentRoot, params) +} + +// Writes a state diff from the current block, parent state root, and provided params +func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params Params) error { + var totalDifficulty *big.Int + var receipts types.Receipts + if params.IncludeTD { + totalDifficulty = sds.BlockChain.GetTdByHash(block.Hash()) + } + if params.IncludeReceipts { + receipts = sds.BlockChain.GetReceiptsByHash(block.Hash()) + } + tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty) + // defer handling of commit/rollback for any return case + defer tx.Close() + output := func(node sdtypes.StateNode) error { + return sds.indexer.PushStateNode(tx, node) + return nil + } + _, err = sds.Builder.WriteStateDiffObject(StateRoots{ + NewStateRoot: block.Root(), + OldStateRoot: parentRoot, + }, params, output) + + // allow dereferencing of parent, keep current locked as it should be the next parent + sds.BlockChain.UnlockTrie(parentRoot) + if err != nil { + return err + } + return nil +} diff --git a/statediff/service_test.go b/statediff/service_test.go index f0262e11a241..0f161757af19 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/statediff" + statediff "github.com/ethereum/go-ethereum/statediff" "github.com/ethereum/go-ethereum/statediff/testhelpers/mocks" ) diff --git a/statediff/testhelpers/mocks/builder.go b/statediff/testhelpers/mocks/builder.go index 5e611aefa986..8abc622b441c 100644 --- a/statediff/testhelpers/mocks/builder.go +++ b/statediff/testhelpers/mocks/builder.go @@ -19,12 +19,14 @@ package mocks import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff" + sdtypes "github.com/ethereum/go-ethereum/statediff/types" ) // Builder is a mock state diff builder type Builder struct { Args statediff.Args Params statediff.Params + StateRoots statediff.StateRoots stateDiff statediff.StateObject block *types.Block stateTrie statediff.StateObject @@ -39,6 +41,14 @@ func (builder *Builder) BuildStateDiffObject(args statediff.Args, params statedi return builder.stateDiff, builder.builderError } +// BuildStateDiffObject mock method +func (builder *Builder) WriteStateDiffObject(args statediff.StateRoots, params statediff.Params, output sdtypes.StateNodeSink) error { + builder.StateRoots = args + builder.Params = params + + return builder.builderError +} + // BuildStateTrieObject mock method func (builder *Builder) BuildStateTrieObject(block *types.Block) (statediff.StateObject, error) { builder.block = block diff --git a/statediff/testhelpers/mocks/service.go b/statediff/testhelpers/mocks/service.go index 6f04da8fb9b7..3805e3d9f9f9 100644 --- a/statediff/testhelpers/mocks/service.go +++ b/statediff/testhelpers/mocks/service.go @@ -129,6 +129,19 @@ func (sds *MockStateDiffService) StateDiffAt(blockNumber uint64, params statedif return sds.processStateDiff(currentBlock, parentBlock.Root(), params) } +// WriteStateDiffAt mock method +func (sds *MockStateDiffService) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error { + currentBlock := sds.BlockChain.GetBlockByNumber(blockNumber) + log.Info(fmt.Sprintf("sending state diff at %d", blockNumber)) + parentRoot := common.Hash{} + if blockNumber != 0 { + parentBlock := sds.BlockChain.GetBlockByHash(currentBlock.ParentHash()) + parentRoot = parentBlock.Root() + } + _, err := sds.processStateDiff(currentBlock, parentRoot, params) + return err +} + // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params func (sds *MockStateDiffService) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) { stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{ diff --git a/statediff/types.go b/statediff/types.go index f3ff8818adf7..63491ccd8378 100644 --- a/statediff/types.go +++ b/statediff/types.go @@ -23,9 +23,9 @@ import ( "encoding/json" "math/big" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/statediff/types" ) // Subscription struct holds our subscription channels @@ -52,6 +52,10 @@ type Args struct { BlockNumber *big.Int } +type StateRoots struct { + OldStateRoot, NewStateRoot common.Hash +} + // Payload packages the data to send to statediff subscriptions type Payload struct { BlockRlp []byte `json:"blockRlp"` @@ -85,7 +89,7 @@ func (sd *Payload) Encode() ([]byte, error) { type StateObject struct { BlockNumber *big.Int `json:"blockNumber" gencodec:"required"` BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Nodes []StateNode `json:"nodes" gencodec:"required"` + Nodes []types.StateNode `json:"nodes" gencodec:"required"` CodeAndCodeHashes []CodeAndCodeHash `json:"codeMapping"` } @@ -96,42 +100,14 @@ type CodeAndCodeHash struct { Code []byte `json:"code"` } -// StateNode holds the data for a single state diff node -type StateNode struct { - NodeType NodeType `json:"nodeType" gencodec:"required"` - Path []byte `json:"path" gencodec:"required"` - NodeValue []byte `json:"value" gencodec:"required"` - StorageNodes []StorageNode `json:"storage"` - LeafKey []byte `json:"leafKey"` -} - -// StorageNode holds the data for a single storage diff node -type StorageNode struct { - NodeType NodeType `json:"nodeType" gencodec:"required"` - Path []byte `json:"path" gencodec:"required"` - NodeValue []byte `json:"value" gencodec:"required"` - LeafKey []byte `json:"leafKey"` -} - // AccountMap is a mapping of hex encoded path => account wrapper type AccountMap map[string]accountWrapper // accountWrapper is used to temporary associate the unpacked node with its raw values type accountWrapper struct { Account *state.Account - NodeType NodeType + NodeType types.NodeType Path []byte NodeValue []byte LeafKey []byte } - -// NodeType for explicitly setting type of node -type NodeType string - -const ( - Unknown NodeType = "Unknown" - Leaf NodeType = "Leaf" - Extension NodeType = "Extension" - Branch NodeType = "Branch" - Removed NodeType = "Removed" // used to represent pathes which have been emptied -) diff --git a/statediff/types/types.go b/statediff/types/types.go new file mode 100644 index 000000000000..3d19ca50be54 --- /dev/null +++ b/statediff/types/types.go @@ -0,0 +1,51 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a batch of utility type declarations used by the tests. As the node +// operates on unique types, a lot of them are needed to check various features. + +package types + +// NodeType for explicitly setting type of node +type NodeType string + +const ( + Unknown NodeType = "Unknown" + Leaf NodeType = "Leaf" + Extension NodeType = "Extension" + Branch NodeType = "Branch" + Removed NodeType = "Removed" // used to represent pathes which have been emptied +) + +// StateNode holds the data for a single state diff node +type StateNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + StorageNodes []StorageNode `json:"storage"` + LeafKey []byte `json:"leafKey"` +} + +// StorageNode holds the data for a single storage diff node +type StorageNode struct { + NodeType NodeType `json:"nodeType" gencodec:"required"` + Path []byte `json:"path" gencodec:"required"` + NodeValue []byte `json:"value" gencodec:"required"` + LeafKey []byte `json:"leafKey"` +} + +type StateNodeSink func(StateNode) error +type StorageNodeSink func(StorageNode) error