Use single codepath for sending transactions to a local and remote no…

…des (#527)

- new EthereumTransactor that provides higher level API for working with ethereum network, and it is fully conformant with ethclient
- new test rpc service that improves flexibility and coverage of txqueue manager tests
- run complete transaction sequantially for each address
- go-ethereum: Get pending nonce from transaction pool
- add a patch with getting nonce from transaction pool

.travis.yml

@@ -23,7 +23,7 @@ jobs:
     # using fork == false may be preferred as it would allow PRs from the origin
     # to still run, but currently does not work due to a bug
     if: type != pull_request
-    script: travis_wait 300 make test-e2e networkid=4
+    script: make test-e2e networkid=4
 cache:
   directories:
   - ".ethereumtest/Mainnet"

Makefile

@@ -113,6 +113,7 @@ mock: ##@other Regenerate mocks
 	mockgen -source=geth/mailservice/mailservice.go -destination=geth/mailservice/mailservice_mock.go -package=mailservice
 	mockgen -source=geth/common/notification.go -destination=geth/common/notification_mock.go -package=common -imports fcm=github.com/NaySoftware/go-fcm
 	mockgen -source=geth/notification/fcm/client.go -destination=geth/notification/fcm/client_mock.go -package=fcm -imports fcm=github.com/NaySoftware/go-fcm
+	mockgen -source=geth/txqueue/fake/txservice.go -destination=geth/txqueue/fake/mock.go -package=fake
 
 test: test-unit-coverage ##@tests Run basic, short tests during development
 

geth-patches/0008-tx-pool-nonce.patch

@@ -0,0 +1,19 @@
+diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go
+index 362379cc..6e12e500 100644
+--- a/internal/ethapi/api.go
++++ b/internal/ethapi/api.go
+@@ -956,6 +956,14 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx cont
+
+ // GetTransactionCount returns the number of transactions the given address has sent for the given block number
+ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Uint64, error) {
++	// go-ethereum issue https://github.com/ethereum/go-ethereum/issues/2880
++	if blockNr == rpc.PendingBlockNumber {
++		nonce, err := s.b.GetPoolNonce(ctx, address)
++		if err != nil {
++			return nil, err
++		}
++		return (*hexutil.Uint64)(&nonce), nil
++	}
+ 	state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr)
+ 	if state == nil || err != nil {
+ 		return nil, err

geth-patches/README.md

@@ -16,6 +16,7 @@ We try to minimize number and amount of changes in those patches as much as poss
  - `0005-whisper-delivery.patch` - adds support for logs/traces of Whisper traffic (questionable, nobody used this functionality so far)
  - `0006-latest-cht.patch` – updates CHT root hashes, should be updated regularly to keep sync fast, until proper Trusted Checkpoint sync is not implemented as part of LES/2 protocol.
  - `0007-README.patch` — update upstream README.md.
+ - `0008-tx-pool-nonce.patch` - On GetTransactionCount request with PendingBlockNumber get the nonce from transaction pool
 
 # Updating upstream version
 

geth/api/backend.go

@@ -90,8 +90,6 @@ func (m *StatusBackend) StartNode(config *params.NodeConfig) (<-chan struct{}, e
 		return nil, err
 	}
 
-	m.txQueueManager.Start()
-
 	m.nodeReady = make(chan struct{}, 1)
 	go m.onNodeStart(nodeStarted, m.nodeReady) // waits on nodeStarted, writes to backendReady
 
@@ -102,6 +100,10 @@ func (m *StatusBackend) StartNode(config *params.NodeConfig) (<-chan struct{}, e
 func (m *StatusBackend) onNodeStart(nodeStarted <-chan struct{}, backendReady chan struct{}) {
 	<-nodeStarted
 
+	// tx queue manager should be started after node is started, it depends
+	// on rpc client being created
+	m.txQueueManager.Start()
+
 	if err := m.registerHandlers(); err != nil {
 		log.Error("Handler registration failed", "err", err)
 	}

geth/txqueue/addrlock.go

@@ -0,0 +1,39 @@
+// copy of go-ethereum/internal/ethapi/addrlock.go
+
+package txqueue
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type AddrLocker struct {
+	mu    sync.Mutex
+	locks map[common.Address]*sync.Mutex
+}
+
+// lock returns the lock of the given address.
+func (l *AddrLocker) lock(address common.Address) *sync.Mutex {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.locks == nil {
+		l.locks = make(map[common.Address]*sync.Mutex)
+	}
+	if _, ok := l.locks[address]; !ok {
+		l.locks[address] = new(sync.Mutex)
+	}
+	return l.locks[address]
+}
+
+// LockAddr locks an account's mutex. This is used to prevent another tx getting the
+// same nonce until the lock is released. The mutex prevents the (an identical nonce) from
+// being read again during the time that the first transaction is being signed.
+func (l *AddrLocker) LockAddr(address common.Address) {
+	l.lock(address).Lock()
+}
+
+// UnlockAddr unlocks the mutex of the given account.
+func (l *AddrLocker) UnlockAddr(address common.Address) {
+	l.lock(address).Unlock()
+}

geth/txqueue/ethtxclient.go

@@ -0,0 +1,93 @@
+package txqueue
+
+import (
+	"context"
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/status-im/status-go/geth/rpc"
+)
+
+// EthTransactor provides methods to create transactions for ethereum network.
+type EthTransactor interface {
+	PendingNonceAt(ctx context.Context, account common.Address) (uint64, error)
+	ethereum.GasEstimator
+	ethereum.GasPricer
+	ethereum.TransactionSender
+}
+
+// EthTxClient wraps common API methods that are used to send transaction.
+type EthTxClient struct {
+	c *rpc.Client
+}
+
+func NewEthTxClient(client *rpc.Client) *EthTxClient {
+	return &EthTxClient{c: client}
+}
+
+// PendingNonceAt returns the account nonce of the given account in the pending state.
+// This is the nonce that should be used for the next transaction.
+func (ec *EthTxClient) PendingNonceAt(ctx context.Context, account common.Address) (uint64, error) {
+	var result hexutil.Uint64
+	err := ec.c.CallContext(ctx, &result, "eth_getTransactionCount", account, "pending")
+	return uint64(result), err
+}
+
+// SuggestGasPrice retrieves the currently suggested gas price to allow a timely
+// execution of a transaction.
+func (ec *EthTxClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
+	var hex hexutil.Big
+	if err := ec.c.CallContext(ctx, &hex, "eth_gasPrice"); err != nil {
+		return nil, err
+	}
+	return (*big.Int)(&hex), nil
+}
+
+// EstimateGas tries to estimate the gas needed to execute a specific transaction based on
+// the current pending state of the backend blockchain. There is no guarantee that this is
+// the true gas limit requirement as other transactions may be added or removed by miners,
+// but it should provide a basis for setting a reasonable default.
+func (ec *EthTxClient) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (*big.Int, error) {
+	var hex hexutil.Big
+	err := ec.c.CallContext(ctx, &hex, "eth_estimateGas", toCallArg(msg))
+	if err != nil {
+		return nil, err
+	}
+	return (*big.Int)(&hex), nil
+}
+
+// SendTransaction injects a signed transaction into the pending pool for execution.
+//
+// If the transaction was a contract creation use the TransactionReceipt method to get the
+// contract address after the transaction has been mined.
+func (ec *EthTxClient) SendTransaction(ctx context.Context, tx *types.Transaction) error {
+	data, err := rlp.EncodeToBytes(tx)
+	if err != nil {
+		return err
+	}
+	return ec.c.CallContext(ctx, nil, "eth_sendRawTransaction", common.ToHex(data))
+}
+
+func toCallArg(msg ethereum.CallMsg) interface{} {
+	arg := map[string]interface{}{
+		"from": msg.From,
+		"to":   msg.To,
+	}
+	if len(msg.Data) > 0 {
+		arg["data"] = hexutil.Bytes(msg.Data)
+	}
+	if msg.Value != nil {
+		arg["value"] = (*hexutil.Big)(msg.Value)
+	}
+	if msg.Gas != nil {
+		arg["gas"] = (*hexutil.Big)(msg.Gas)
+	}
+	if msg.GasPrice != nil {
+		arg["gasPrice"] = (*hexutil.Big)(msg.GasPrice)
+	}
+	return arg
+}

geth/txqueue/fake/txservice.go

@@ -0,0 +1,40 @@
+package fake
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/golang/mock/gomock"
+)
+
+func NewTestServer(ctrl *gomock.Controller) (*rpc.Server, *MockFakePublicTransactionPoolAPI) {
+	srv := rpc.NewServer()
+	svc := NewMockFakePublicTransactionPoolAPI(ctrl)
+	if err := srv.RegisterName("eth", svc); err != nil {
+		panic(err)
+	}
+	return srv, svc
+}
+
+// CallArgs copied from module go-ethereum/internal/ethapi
+type CallArgs struct {
+	From     common.Address  `json:"from"`
+	To       *common.Address `json:"to"`
+	Gas      hexutil.Big     `json:"gas"`
+	GasPrice hexutil.Big     `json:"gasPrice"`
+	Value    hexutil.Big     `json:"value"`
+	Data     hexutil.Bytes   `json:"data"`
+}
+
+// FakePublicTransactionPoolAPI used to generate mock by mockgen util.
+// This was done because PublicTransactionPoolAPI is located in internal/ethapi module
+// and there is no easy way to generate mocks from internal modules.
+type FakePublicTransactionPoolAPI interface {
+	GasPrice(ctx context.Context) (*big.Int, error)
+	EstimateGas(ctx context.Context, args CallArgs) (*hexutil.Big, error)
+	GetTransactionCount(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Uint64, error)
+	SendRawTransaction(ctx context.Context, encodedTx hexutil.Bytes) (common.Hash, error)
+}