Update transactions, receipts and logs queries (#173)

* Update queries to fetch data from cids tables

* Fix eth_getTransactionByHash to return tx from canonical block

* Fetch transaction by hash in a single query

* Update queries to fetch data from IPLD blocks table

* Avoid NULL result from canonical block hash query

* Update GitHub workflow and script to run unit tests

* Avoid usage of LIMIT 1 in queries for GQL API

* Fetch IPLD data separately when retrieving latest account data or storage value

* Upgrade geth dependency

* Update GitHub workflow

.github/workflows/on-pr-publish.yaml

@@ -28,9 +28,9 @@ jobs:
       BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }}
       BUILD_KEY: ${{ secrets.BUILD_KEY }}
     with:
-      STACK_ORCHESTRATOR_REF: "382aca8e42bc5e33f301f77cdd2e09cc80602fc3"
-      GO_ETHEREUM_REF: "c265fdc30915e01bb633203acbbd9d5009a7ddf2"
-      IPLD_ETH_DB_REF: "4e948c58ce20c20ab633289f986d2ed2a1fe02ec"
+      STACK_ORCHESTRATOR_REF: "f2fd766f5400fcb9eb47b50675d2e3b1f2753702"
+      GO_ETHEREUM_REF: "7b4ef34de2b9469c3f82972b60e38b34c99c5382"
+      IPLD_ETH_DB_REF: "b59505eab252670c622b42ce60621e9747fb64f9"
   build:
     name: Run docker build
     runs-on: ubuntu-latest

.github/workflows/run_unit_test.sh

@@ -12,31 +12,17 @@ cd ipld-eth-server
 ## Remove the branch and github related info. This way future runs wont be confused.
 rm -f /tmp/git_head_ref /tmp/git_repository
 
-# Setup the DB
-cd $temp_dir
-git clone "https://github.com/vulcanize/ipld-eth-db.git"; cd ipld-eth-db; git checkout $(cat /tmp/ipld_eth_db_ref)
-
-# Spin Up DB using Stack Orchestrator
-cd $temp_dir
-git clone "https://github.com/vulcanize/stack-orchestrator.git"; cd stack-orchestrator; git checkout $(cat /tmp/stack_orchestrator_ref)
-
-cd ${temp_dir}/stack-orchestrator
-echo vulcanize_ipld_eth_db=${temp_dir}/ipld-eth-db > ./config.sh
-
-## Remove existing containers if they are present
-docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans;
-
-trap 'cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans; ' SIGINT SIGTERM
-docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh up -d
+# Spin up DB and run migrations
+docker-compose up -d migrations ipld-eth-db
+trap "docker-compose down -v --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR
+sleep 30
 
 # Remove old logs so there's no confusion, then run test
 rm -f /tmp/test.log /tmp/return_test.txt
-cd ${temp_dir}/ipld-eth-server
 PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=localhost DATABASE_NAME=vulcanize_testing make test >  /tmp/test.log
 echo $? > /tmp/return_test.txt
 
 # Clean up
-
-cd ${temp_dir}/stack-orchestrator; docker-compose -f docker/local/docker-compose-db-sharding.yml --env-file ./config.sh down -v --remove-orphans
+docker-compose down -v --remove-orphans
 cd $start_dir
 rm -fr $temp_dir

.github/workflows/tests.yaml

@@ -46,24 +46,18 @@ jobs:
       - name: Output variables to files
         run: |
           echo $GITHUB_REPOSITORY > /tmp/git_repository
-          echo ${{ inputs.IPLD_ETH_DB_REF }} > /tmp/ipld_eth_db_ref
-          echo ${{ inputs.STACK_ORCHESTRATOR_REF }} > /tmp/stack_orchestrator_ref
           [ -z "$GITHUB_HEAD_REF" ] && echo $GITHUB_REF_NAME > /tmp/git_head_ref || echo $GITHUB_HEAD_REF > /tmp/git_head_ref
           echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key
           echo ${{ env.BUILD_KEY }} >> /tmp/key
           echo "-----END OPENSSH PRIVATE KEY-----" >> /tmp/key
           chmod 400 /tmp/key
           cat /tmp/git_repository
           cat /tmp/git_head_ref
-          cat /tmp/stack_orchestrator_ref
-          cat /tmp/ipld_eth_db_ref
 
       - name: Raw SCP
         run: |
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_repository ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_repository
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_head_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_head_ref
-          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/stack_orchestrator_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/stack_orchestrator_ref
-          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/ipld_eth_db_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/ipld_eth_db_ref
           scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key .github/workflows/run_unit_test.sh ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/run_unit_test.sh
 
       - name: Trigger Unit Test
@@ -121,12 +115,11 @@ jobs:
           echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
           echo vulcanize_ipld_eth_server=$GITHUB_WORKSPACE/ipld-eth-server/ >> ./config.sh
           echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-server/test/contract >> ./config.sh
+          echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
           echo db_write=$DB_WRITE >> ./config.sh
           echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
           echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
           echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
-          echo watched_address_gap_filler_enabled=false >> ./config.sh
-          echo watched_address_gap_filler_interval=5 >> ./config.sh
           cat ./config.sh
       - name: Build geth
         run: |
@@ -190,12 +183,11 @@ jobs:
           echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ >> ./config.sh
           echo vulcanize_ipld_eth_server=$GITHUB_WORKSPACE/ipld-eth-server/ >> ./config.sh
           echo vulcanize_test_contract=$GITHUB_WORKSPACE/ipld-eth-server/test/contract >>./config.sh
+          echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> ./config.sh
           echo db_write=$DB_WRITE >> ./config.sh
           echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
           echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
           echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
-          echo watched_address_gap_filler_enabled=false >> ./config.sh
-          echo watched_address_gap_filler_interval=5 >> ./config.sh
           cat ./config.sh
       - name: Build geth
         run: |

README.md

@@ -22,13 +22,13 @@ Additional, unique endpoints are exposed which utilize the new indexes and state
 
 ## Dependencies
 Minimal build dependencies
-* Go (1.13)
+* Go (1.18)
 * Git
 * GCC compiler
 * This repository
 
 External dependency
-* Postgres database populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)
+* Postgres database populated by [ipld-eth-db](https://github.com/vulcanize/ipld-eth-db)
 
 ## Install
 Start by downloading ipld-eth-server and moving into the repo:

docker-compose.yml

@@ -1,18 +1,29 @@
 version: '3.2'
 
 services:
-  ipld-eth-db:
+  migrations:
     restart: on-failure
     depends_on:
-      - access-node
-    image: vulcanize/ipld-eth-db:v4.1.1-alpha
+      - ipld-eth-db
+    image: vulcanize/ipld-eth-db:v4.2.0-alpha
     environment:
       DATABASE_USER: "vdbm"
-      DATABASE_NAME: "vulcanize_testing_v4"
+      DATABASE_NAME: "vulcanize_testing"
       DATABASE_PASSWORD: "password"
-      DATABASE_HOSTNAME: "access-node"
+      DATABASE_HOSTNAME: "ipld-eth-db"
       DATABASE_PORT: 5432
 
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - "127.0.0.1:8077:5432"
+
   eth-server:
     restart: unless-stopped
     depends_on:
@@ -44,20 +55,5 @@ services:
     ports:
      - "127.0.0.1:8081:8081"
 
-  graphql:
-    restart: unless-stopped
-    depends_on:
-      - ipld-eth-db
-    image: vulcanize/postgraphile:v1.0.1
-    environment:
-      - PG_HOST=db
-      - PG_PORT=5432
-      - PG_DATABASE=vulcanize_public
-      - PG_USER=vdbm
-      - PG_PASSWORD=password
-      - SCHEMA=public,eth
-    ports:
-      - "127.0.0.1:5000:5000"
-
 volumes:
   vdb_db_eth_server:

go.mod

@@ -289,4 +289,4 @@ require (
 	lukechampine.com/blake3 v1.1.6 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha
+replace github.com/ethereum/go-ethereum v1.10.19 => github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha

go.sum

@@ -1662,8 +1662,8 @@ github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha h1:sDDK4eOdW3JEds+
 github.com/vulcanize/eth-ipfs-state-validator/v4 v4.0.3-alpha/go.mod h1:/pHfZd1IWsSTpCtGq6nnzUZBAkLV+zMrRh6Z3Hr3NFc=
 github.com/vulcanize/gap-filler v0.4.0 h1:5VD9PG7UrjEub4rLxZmstWoHnBnVtXz9silIVdrnTsM=
 github.com/vulcanize/gap-filler v0.4.0/go.mod h1:5awUyotIoJi6AuG0JPEm7SIwFZBD7Ecg0I8x7CdxcHI=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha h1:xD4fA2khoAnhBEk84JwrIEGvQCndVXpQGv5n7a9cgwc=
-github.com/vulcanize/go-ethereum v1.10.19-statediff-4.0.2-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha h1:8ge2ban6t/e53XDwe6s28jcCevT7Ggo51lNJ0Eo1PgA=
+github.com/vulcanize/go-ethereum v1.10.19-statediff-4.1.0-alpha/go.mod h1:5tMN+CDbK/qI2UlfN307HJykDmVIOCB1FM5RcHK9Kp8=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha h1:xak1uYmFWqJ2Hz3pM+0jDcqdlwYwRWeSkQV6B8IxD/0=
 github.com/vulcanize/ipfs-ethdb/v4 v4.0.2-alpha/go.mod h1:pHbLbW4Hk1IFpxrY9yi50IuoPPzmSY7lwOqpFAa369k=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=

pkg/eth/backend.go

@@ -52,22 +52,22 @@ import (
 )
 
 var (
-	errPendingBlockNumber  = errors.New("pending block number not supported")
-	errNegativeBlockNumber = errors.New("negative block number not supported")
-	errHeaderHashNotFound  = errors.New("header for hash not found")
-	errHeaderNotFound      = errors.New("header not found")
+	errPendingBlockNumber     = errors.New("pending block number not supported")
+	errNegativeBlockNumber    = errors.New("negative block number not supported")
+	errHeaderHashNotFound     = errors.New("header for hash not found")
+	errHeaderNotFound         = errors.New("header not found")
+	errMultipleHeadersForHash = errors.New("more than one headers for the given hash")
+	errTxHashNotFound         = errors.New("transaction for hash not found")
+	errTxHashInMultipleBlocks = errors.New("transaction for hash found in more than one canonical block")
 
 	// errMissingSignature is returned if a block's extra-data section doesn't seem
 	// to contain a 65 byte secp256k1 signature.
 )
 
 const (
-	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids
-									INNER JOIN public.blocks ON (
-										header_cids.mh_key = blocks.key
-										AND header_cids.block_number = blocks.block_number
-									)
-									WHERE block_hash = (SELECT canonical_header_hash($1))`
+	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash
+									FROM canonical_header_hash($1) AS block_hash
+									WHERE block_hash IS NOT NULL`
 	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
 									INNER JOIN public.blocks ON (
 										header_cids.mh_key = blocks.key
@@ -76,13 +76,12 @@ const (
 									WHERE block_hash = (SELECT canonical_header_hash($1))`
 	RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids
 			WHERE header_cids.block_hash = $1`
-	RetrieveRPCTransaction = `SELECT blocks.data, block_hash, transaction_cids.block_number, index
-			FROM public.blocks, eth.transaction_cids, eth.header_cids
+	RetrieveRPCTransaction = `SELECT blocks.data, header_id, transaction_cids.block_number, index
+			FROM public.blocks, eth.transaction_cids
 			WHERE blocks.key = transaction_cids.mh_key
 			AND blocks.block_number = transaction_cids.block_number
-			AND transaction_cids.header_id = header_cids.block_hash
-			AND transaction_cids.block_number = header_cids.block_number
-			AND transaction_cids.tx_hash = $1`
+			AND transaction_cids.tx_hash = $1
+			AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))`
 	RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
 											WHERE state_accounts.header_id = state_cids.header_id
 											AND state_accounts.state_path = state_cids.state_path
@@ -524,20 +523,30 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 // GetTransaction retrieves a tx by hash
 // It also returns the blockhash, blocknumber, and tx index associated with the transaction
 func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) {
-	var tempTxStruct struct {
+	type txRes struct {
 		Data        []byte `db:"data"`
-		BlockHash   string `db:"block_hash"`
+		HeaderID    string `db:"header_id"`
 		BlockNumber uint64 `db:"block_number"`
 		Index       uint64 `db:"index"`
 	}
-	if err := b.DB.Get(&tempTxStruct, RetrieveRPCTransaction, txHash.String()); err != nil {
+	var res = make([]txRes, 0)
+	if err := b.DB.Select(&res, RetrieveRPCTransaction, txHash.String()); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
+
+	if len(res) == 0 {
+		return nil, common.Hash{}, 0, 0, errTxHashNotFound
+	} else if len(res) > 1 {
+		// a transaction can be part of a only one canonical block
+		return nil, common.Hash{}, 0, 0, errTxHashInMultipleBlocks
+	}
+
 	var transaction types.Transaction
-	if err := transaction.UnmarshalBinary(tempTxStruct.Data); err != nil {
+	if err := transaction.UnmarshalBinary(res[0].Data); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
-	return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil
+
+	return &transaction, common.HexToHash(res[0].HeaderID), res[0].BlockNumber, res[0].Index, nil
 }
 
 // GetReceipts retrieves receipts for provided block hash