Merge pull request #16 from vulcanize/update_deps

upgrade ipld-eth-indexer from v0.2.0-alpha to v0.5.0-alpha

cmd/subscribe.go

@@ -28,8 +28,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/client"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
@@ -83,7 +81,7 @@ func subscribe() {
 				logWithCommand.Error(payload.Err)
 				continue
 			}
-			var ethData eth2.IPLDs
+			var ethData eth.IPLDs
 			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
 				logWithCommand.Error(err)
 				continue

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+ALTER TABLE public.nodes
+ADD COLUMN chain_id INTEGER DEFAULT 1;
+
+ALTER TABLE public.nodes
+DROP CONSTRAINT node_uc;
+
+ALTER TABLE public.nodes
+ADD CONSTRAINT node_uc
+UNIQUE (genesis_block, network_id, node_id, chain_id);
+
+-- +goose Down
+ALTER TABLE public.nodes
+DROP CONSTRAINT node_uc;
+
+ALTER TABLE public.nodes
+ADD CONSTRAINT node_uc
+UNIQUE (genesis_block, network_id, node_id);
+
+ALTER TABLE public.nodes
+DROP COLUMN chain_id;

db/schema.sql

@@ -361,7 +361,8 @@ CREATE TABLE public.nodes (
     client_name character varying,
     genesis_block character varying(66),
     network_id character varying,
-    node_id character varying(128)
+    node_id character varying(128),
+    chain_id integer
 );
 
 
@@ -595,7 +596,7 @@ ALTER TABLE ONLY public.goose_db_version
 --
 
 ALTER TABLE ONLY public.nodes
-    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
+    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id);
 
 
 --

go.mod

@@ -4,20 +4,30 @@ go 1.13
 
 require (
 	github.com/ethereum/go-ethereum v1.9.11
+	github.com/go-sql-driver/mysql v1.5.0 // indirect
+	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-cid v0.0.5
 	github.com/ipfs/go-ipfs-blockstore v1.0.0
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.5.2
+	github.com/lib/pq v1.8.0
+	github.com/mattn/go-sqlite3 v1.14.4 // indirect
 	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.12.1
+	github.com/nxadm/tail v1.4.5 // indirect
+	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.1
+	github.com/pressly/goose v2.6.0+incompatible // indirect
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha
+	github.com/vulcanize/ipld-eth-indexer v0.5.0-alpha
 	github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha
+	github.com/ziutek/mymysql v1.5.4 // indirect
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
+	golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13 // indirect
+	golang.org/x/tools v0.0.0-20201019175715-b894a3290fff // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 )
 
 replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5

pkg/eth/cid_retriever.go

@@ -36,7 +36,7 @@ import (
 type Retriever interface {
 	RetrieveFirstBlockNumber() (int64, error)
 	RetrieveLastBlockNumber() (int64, error)
-	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error)
+	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error)
 }
 
 // CIDRetriever satisfies the CIDRetriever interface for ethereum
@@ -66,7 +66,7 @@ func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
 }
 
 // Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error) {
+func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error) {
 	log.Debug("retrieving cids")
 
 	// Begin new db tx
@@ -91,10 +91,10 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		log.Error("header cid retrieval error")
 		return nil, true, err
 	}
-	cws := make([]eth2.CIDWrapper, len(headers))
+	cws := make([]CIDWrapper, len(headers))
 	empty := true
 	for i, header := range headers {
-		cw := new(eth2.CIDWrapper)
+		cw := new(CIDWrapper)
 		cw.BlockNumber = big.NewInt(blockNumber)
 		if !filter.HeaderFilter.Off {
 			cw.Header = header

pkg/eth/cid_retriever_test.go

@@ -17,6 +17,7 @@
 package eth_test
 
 import (
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/mocks"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -25,7 +26,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
+	mocks2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
@@ -52,7 +53,7 @@ var (
 			Off: true,
 		},
 		ReceiptFilter: eth.ReceiptFilter{
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -94,7 +95,7 @@ var (
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
 				{"0x0000000000000000000000000000000000000000000000000000000000000006"},
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -115,9 +116,9 @@ var (
 		ReceiptFilter: eth.ReceiptFilter{
 			Topics: [][]string{
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks.Address.String() contract receipt
+				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks2.Address.String() contract receipt
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -137,7 +138,7 @@ var (
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000005"}},
-			LogAddresses: []string{mocks.Address.String(), mocks.AnotherAddress.String()},
+			LogAddresses: []string{mocks2.Address.String(), mocks2.AnotherAddress.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -172,7 +173,7 @@ var (
 			Off: true,
 		},
 		TxFilter: eth.TxFilter{
-			Dst: []string{mocks.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
+			Dst: []string{mocks2.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			MatchTxs:     true,
@@ -199,7 +200,7 @@ var (
 			Off: true,
 		},
 		StateFilter: eth.StateFilter{
-			Addresses: []string{mocks.AccountAddresss.Hex()},
+			Addresses: []string{mocks2.AccountAddresss.Hex()},
 		},
 		StorageFilter: eth.StorageFilter{
 			Off: true,
@@ -249,12 +250,12 @@ var _ = Describe("Retriever", func() {
 			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
 			for _, stateNode := range cids[0].StateNodes {
-				if stateNode.CID == mocks.State1CID.String() {
+				if stateNode.CID == mocks2.State1CID.String() {
 					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
 				}
-				if stateNode.CID == mocks.State2CID.String() {
+				if stateNode.CID == mocks2.State2CID.String() {
 					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
@@ -378,9 +379,9 @@ var _ = Describe("Retriever", func() {
 				ID:       cids7[0].StateNodes[0].ID,
 				HeaderID: cids7[0].StateNodes[0].HeaderID,
 				NodeType: 2,
-				StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
-				CID:      mocks.State2CID.String(),
-				MhKey:    mocks.State2MhKey,
+				StateKey: common.BytesToHash(mocks2.AccountLeafKey).Hex(),
+				CID:      mocks2.State2CID.String(),
+				MhKey:    mocks2.State2MhKey,
 				Path:     []byte{'\x0c'},
 			}))
 
@@ -396,15 +397,15 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := mocks2.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -414,7 +415,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := mocks2.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -434,15 +435,15 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := mocks2.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -452,7 +453,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := mocks2.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -468,7 +469,7 @@ var _ = Describe("Retriever", func() {
 })
 
 func newMockBlock(blockNumber uint64) *types.Block {
-	header := mocks.MockHeader
+	header := mocks2.MockHeader
 	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&mocks.MockHeader, mocks.MockTransactions, nil, mocks.MockReceipts)
+	return types.NewBlock(&mocks2.MockHeader, mocks2.MockTransactions, nil, mocks2.MockReceipts)
 }

pkg/eth/filterer.go

@@ -33,7 +33,7 @@ import (
 
 // Filterer interface for substituing mocks in tests
 type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error)
+	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
 }
 
 // ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -45,9 +45,9 @@ func NewResponseFilterer() *ResponseFilterer {
 }
 
 // Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error) {
+func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
 	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
-		response := new(eth.IPLDs)
+		response := new(IPLDs)
 		response.TotalDifficulty = payload.TotalDifficulty
 		if err := s.filterHeaders(filter.HeaderFilter, response, payload); err != nil {
 			return nil, err
@@ -72,7 +72,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
 	return nil, nil
 }
 
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
 	if !headerFilter.Off {
 		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
 		if err != nil {
@@ -114,7 +114,7 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *eth.IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
 	var trxHashes []common.Hash
 	if !trxFilter.Off {
 		trxLen := len(payload.Block.Body().Transactions)
@@ -162,7 +162,7 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 	return false
 }
 
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *eth.IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
 		for i, receipt := range payload.Receipts {
@@ -252,9 +252,9 @@ func slicesShareString(slice1, slice2 []string) int {
 }
 
 // filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
-	response.StateNodes = make([]eth.StateNode, 0, len(payload.StateNodes))
-	response.StorageNodes = make([]eth.StorageNode, 0)
+func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
+	response.StorageNodes = make([]StorageNode, 0)
 	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
 	for i, addr := range stateFilter.Addresses {
 		stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
@@ -274,7 +274,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 				if err != nil {
 					return err
 				}
-				response.StateNodes = append(response.StateNodes, eth.StateNode{
+				response.StateNodes = append(response.StateNodes, StateNode{
 					StateLeafKey: stateNode.LeafKey,
 					Path:         stateNode.Path,
 					IPLD: ipfs.BlockModel{
@@ -292,7 +292,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 					if err != nil {
 						return err
 					}
-					response.StorageNodes = append(response.StorageNodes, eth.StorageNode{
+					response.StorageNodes = append(response.StorageNodes, StorageNode{
 						StateLeafKey:   stateNode.LeafKey,
 						StorageLeafKey: storageNode.LeafKey,
 						IPLD: ipfs.BlockModel{