Proxy v1 (ethereum#515)

* added proxy server

* added isproxy field to peer objects

* relay messages proxy receives to validator

* Add addSentry RPC

* Add removeSentry RPC

* Add admin.sentryInfo RPC

* Add --proxied flag

* Don't consider sentry nodes as contributing toward the max peer count

* Don't assume a sentry will never be a validator

* Remove proxied check from api

* Go back to assuming sentry will not be a validator

* Fix eth tests

* Fix eth test lint

* Add max sentry node count as 1

* wip, add val_enode_share

* Remove redundant proxied check

* Add warning if proxied but not mining

* Wip, messages now are received by sentry

* Clean up, add signatures to val enode share msg

* Add sharedValidatorEnode struct

* Use received shared validator enodes, add proxied check for sharing

* Clean up, remove proxied as a variable in istanbul backend

* run go fmt

* Fix tests

* Add val enode share test

* Change GlobalIsSet -> GlobalBool

* Change to val enode share to every minute, fix comment

* Read lock when reading val enode table entries

* got blockchain syncing done via the sentry

* Refactor val enode share to use istanbul message struct

* changes for relayer consensus message handling

* Refactor announce to use istanbul message struct

* peer.Send done in a goroutine

* Add better logging

* Proxied validators do not peer with validators, sentries do

* Don't refresh val set peers if proxy, lock once when upserting upon handling table message

* Fix lint, tests

* fixed lint errors

* made the sentry and proxied command line argument more consistent

* got announce and val_enode_share msgs working

* forwarding of consensus messages now working

* fix the old block seal

* fixed comment

* added test case for 'old block' preprepare

* simplified the logic

* refactored sentry related command line args and moved addsentry and removesentry rpc api to istanbul package

* refactoring the p2p server to remove sentry and validator connections

* moved all validator and sentry peer management from p2p to istanbul

* fixed bugs

* removed the refactoring of core.finalizeMessage, backend.Broadcast, backend.Gossip.  Leave it for later

* changed proxy listen port flag to listen endpoint

* changes in the test files for the new addpeer and addtrusted p2p.server apis

* changed order of params in Gossip to be same as original ordering

* fixed p2p peer test for the new p2p server interface

* lint fixes

* changes to get istanbul related unit tests to pass

* some changes for the unit tests

* removed some debug logging and added some comment

* added some comments to clarify the need for istanbul.backend.NewWork and istanbul.backend.NewChainHead

* set maxpeers to 1 for the internal server interface

* created istanbul forward message

* removed unneeded whitespace

* handled case of no sentry node set in unregisteredpeer

* got sigint to work

* added support for setting sentry nodes via command line

* fixed a bug

* addressed PR comments

* fixed a big and addressed more PR comments

* will now send an announce message at start of new epoch

* refactor istanbul message (ethereum#583)

* refactored istanbul message

* addressed PR comments

* addressed PR comments

* addressed more PR comments

* addressed more PR comments

* change p2p static and trusted peer labels to purposes

* got proxy end to end test working

* changed the purpose from string type to bitmap based type

* removed redundant adding of active validators to the regAndActiveVal set

* fixed a bug

* re-point celo-monorepo to master

* fixed miner unit test to not create the validatorenode db directory in the miner directory

* addressed PR comments from mcortesi

* handled old announce messages

* fixed a bug

* addressed PR comments

* modified UnionOfSeals to return an error instead of panic-ing

* checked for dup or irrelevant entries when process an announce message

* fixed bugs

* don't regossip announce message if it has dups or irrelevent entries

cmd/faucet/faucet.go

@@ -257,7 +257,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 	for _, boot := range enodes {
 		old, err := enode.ParseV4(boot.String())
 		if err == nil {
-			stack.Server().AddPeer(old)
+			stack.Server().AddPeer(old, p2p.ExplicitStaticPurpose)
 		}
 	}
 	// Attach to the client and retrieve and interesting metadatas

cmd/geth/config.go

@@ -124,6 +124,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 
 	// Apply flags.
 	utils.SetNodeConfig(ctx, &cfg.Node)
+	utils.SetProxyConfig(ctx, &cfg.Node, &cfg.Eth)
 	stack, err := node.New(&cfg.Node)
 	if err != nil {
 		utils.Fatalf("Failed to create the protocol stack: %v", err)

cmd/geth/main.go

@@ -140,8 +140,13 @@ var (
 		utils.IstanbulBlockPeriodFlag,
 		utils.IstanbulProposerPolicyFlag,
 		utils.PingIPFromPacketFlag,
-		utils.UseInMemoryDiscoverTable,
+		utils.UseInMemoryDiscoverTableFlag,
 		utils.VersionCheckFlag,
+		utils.ProxyFlag,
+		utils.ProxyInternalFacingEndpointFlag,
+		utils.ProxiedValidatorAddressFlag,
+		utils.ProxiedFlag,
+		utils.ProxyEnodeURLPairFlag,
 	}
 
 	rpcFlags = []cli.Flag{
@@ -337,12 +342,20 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 			}
 		}
 	}()
+
+	// Miners and proxies only makes sense if a full node is running
+	if ctx.GlobalBool(utils.ProxyFlag.Name) || ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
+		if ctx.GlobalString(utils.SyncModeFlag.Name) != "fast" && ctx.GlobalString(utils.SyncModeFlag.Name) != "full" {
+			utils.Fatalf("Miners and Proxies must be run as a full node")
+		}
+	}
+
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
-		// Mining only makes sense if a full Ethereum node is running
-		if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
-			utils.Fatalf("Light clients do not support mining")
+		if ctx.GlobalBool(utils.ProxyFlag.Name) {
+			utils.Fatalf("Proxies can't mine")
 		}
+
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("Ethereum service not running: %v", err)

cmd/geth/usage.go

@@ -184,7 +184,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
 			utils.PingIPFromPacketFlag,
-			utils.UseInMemoryDiscoverTable,
+			utils.UseInMemoryDiscoverTableFlag,
 		},
 	},
 	{
@@ -232,6 +232,24 @@ var AppHelpFlagGroups = []flagGroup{
 		Name:  "WHISPER (EXPERIMENTAL)",
 		Flags: whisperFlags,
 	},
+	{
+		Name: "ISTANBUL",
+		Flags: []cli.Flag{
+			utils.IstanbulRequestTimeoutFlag,
+			utils.IstanbulBlockPeriodFlag,
+			utils.IstanbulProposerPolicyFlag,
+		},
+	},
+	{
+		Name: "PROXY",
+		Flags: []cli.Flag{
+			utils.ProxyFlag,
+			utils.ProxyInternalFacingEndpointFlag,
+			utils.ProxiedValidatorAddressFlag,
+			utils.ProxiedFlag,
+			utils.ProxyEnodeURLPairFlag,
+		},
+	},
 	{
 		Name: "DEPRECATED",
 		Flags: []cli.Flag{
@@ -247,14 +265,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.VersionCheckFlag,
 		},
 	},
-	{
-		Name: "ISTANBUL",
-		Flags: []cli.Flag{
-			utils.IstanbulRequestTimeoutFlag,
-			utils.IstanbulBlockPeriodFlag,
-			utils.IstanbulProposerPolicyFlag,
-		},
-	},
 }
 
 // byCategory sorts an array of flagGroup by Name in the order

cmd/swarm/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
@@ -316,7 +317,7 @@ func bzzd(ctx *cli.Context) error {
 		s := stack.Server()
 
 		for _, n := range cfg.P2P.BootstrapNodes {
-			s.AddPeer(n)
+			s.AddPeer(n, p2p.ExplicitStaticPurpose)
 		}
 	}()
 

cmd/utils/flags.go

@@ -559,7 +559,7 @@ var (
 		Name:  "ping-ip-from-packet",
 		Usage: "Has the discovery protocol use the IP address given by a ping packet",
 	}
-	UseInMemoryDiscoverTable = cli.BoolFlag{
+	UseInMemoryDiscoverTableFlag = cli.BoolFlag{
 		Name:  "use-in-memory-discovery-table",
 		Usage: "Specifies whether to use an in memory discovery table",
 	}
@@ -661,6 +661,31 @@ var (
 		Usage: "Default minimum difference between two consecutive block's timestamps in seconds",
 		Value: uint64(eth.DefaultConfig.Istanbul.ProposerPolicy),
 	}
+
+	// Proxy node settings
+	ProxyFlag = cli.BoolFlag{
+		Name:  "proxy.proxy",
+		Usage: "Specifies whether this node is a proxy",
+	}
+	ProxyInternalFacingEndpointFlag = cli.StringFlag{
+		Name:  "proxy.internalendpoint",
+		Usage: "Specifies the internal facing endpoint for this proxy to listen to.  The format should be <ip address>:<port>",
+		Value: ":30503",
+	}
+	ProxiedValidatorAddressFlag = cli.StringFlag{
+		Name:  "proxy.proxiedvalidatoraddress",
+		Usage: "Address of the proxied validator",
+	}
+
+	// Proxied validator settings
+	ProxiedFlag = cli.BoolFlag{
+		Name:  "proxy.proxied",
+		Usage: "Specifies whether this validator will be proxied by a proxy node",
+	}
+	ProxyEnodeURLPairFlag = cli.StringFlag{
+		Name:  "proxy.proxyenodeurlpair",
+		Usage: "proxy enode URL pair separated by a semicolon.  The format should be \"<internal facing enode URL>;<external facing enode URL>\"",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1004,13 +1029,14 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 	if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) {
 		cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name)
 	}
-	if ctx.GlobalIsSet(NoDiscoverFlag.Name) || lightClient {
+
+	if ctx.GlobalBool(NoDiscoverFlag.Name) || lightClient {
 		cfg.NoDiscovery = true
 	}
-	if ctx.GlobalIsSet(PingIPFromPacketFlag.Name) {
+	if ctx.GlobalBool(PingIPFromPacketFlag.Name) {
 		cfg.PingIPFromPacket = true
 	}
-	if ctx.GlobalIsSet(UseInMemoryDiscoverTable.Name) {
+	if ctx.GlobalBool(UseInMemoryDiscoverTableFlag.Name) {
 		cfg.UseInMemoryNodeDatabase = true
 	}
 
@@ -1178,7 +1204,77 @@ func setIstanbul(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	cfg.Istanbul.ValidatorEnodeDBPath = stack.ResolvePath(cfg.Istanbul.ValidatorEnodeDBPath)
 }
 
-// checkExclusive verifies that only a single isntance of the provided flags was
+func setProxyP2PConfig(ctx *cli.Context, proxyCfg *p2p.Config) {
+	setNodeKey(ctx, proxyCfg)
+	setNAT(ctx, proxyCfg)
+	if ctx.GlobalIsSet(ProxyInternalFacingEndpointFlag.Name) {
+		proxyCfg.ListenAddr = ctx.GlobalString(ProxyInternalFacingEndpointFlag.Name)
+	}
+
+	proxyCfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
+}
+
+// Set all of the proxy related configurations.
+// These configs span the top level node and istanbul module configuration
+func SetProxyConfig(ctx *cli.Context, nodeCfg *node.Config, ethCfg *eth.Config) {
+	checkExclusive(ctx, ProxyFlag, ProxiedFlag)
+
+	if ctx.GlobalIsSet(ProxyFlag.Name) {
+		nodeCfg.Proxy = ctx.GlobalBool(ProxyFlag.Name)
+		ethCfg.Istanbul.Proxy = ctx.GlobalBool(ProxyFlag.Name)
+
+		// Mining must not be set for proxies
+		if ctx.GlobalIsSet(MiningEnabledFlag.Name) {
+			Fatalf("Option --%s must not be used if option --%s is used", MiningEnabledFlag.Name, ProxyFlag.Name)
+		}
+
+		if !ctx.GlobalIsSet(ProxiedValidatorAddressFlag.Name) {
+			Fatalf("Option --%s must be used if option --%s is used", ProxiedValidatorAddressFlag.Name, ProxyFlag.Name)
+		} else {
+			proxiedValidatorAddress := ctx.String(ProxiedValidatorAddressFlag.Name)
+			if !common.IsHexAddress(proxiedValidatorAddress) {
+				Fatalf("Invalid address used for option --%s", ProxiedValidatorAddressFlag.Name)
+			}
+			ethCfg.Istanbul.ProxiedValidatorAddress = common.HexToAddress(proxiedValidatorAddress)
+		}
+
+		if !ctx.GlobalIsSet(ProxyInternalFacingEndpointFlag.Name) {
+			Fatalf("Option --%s must be used if option --%s is used", ProxyInternalFacingEndpointFlag.Name, ProxyFlag.Name)
+		} else {
+			setProxyP2PConfig(ctx, &nodeCfg.ProxyP2P)
+		}
+	}
+
+	if ctx.GlobalIsSet(ProxiedFlag.Name) {
+		ethCfg.Istanbul.Proxied = ctx.GlobalBool(ProxiedFlag.Name)
+
+		// Mining must be set for proxied nodes
+		if !ctx.GlobalIsSet(MiningEnabledFlag.Name) {
+			Fatalf("Option --%s must be used if option --%s is used", MiningEnabledFlag.Name, ProxiedFlag.Name)
+		}
+
+		if !ctx.GlobalIsSet(ProxyEnodeURLPairFlag.Name) {
+			Fatalf("Option --%s must be used if option --%s is used", ProxyEnodeURLPairFlag.Name, ProxiedFlag.Name)
+		} else {
+			proxyEnodeURLPair := strings.Split(ctx.String(ProxyEnodeURLPairFlag.Name), ";")
+
+			var err error
+			if ethCfg.Istanbul.ProxyInternalFacingNode, err = enode.ParseV4(proxyEnodeURLPair[0]); err != nil {
+				Fatalf("Proxy internal facing enodeURL (%s) invalid with err: %v", proxyEnodeURLPair[0], err)
+			}
+
+			if ethCfg.Istanbul.ProxyExternalFacingNode, err = enode.ParseV4(proxyEnodeURLPair[1]); err != nil {
+				Fatalf("Proxy external facing enodeURL (%s) invalid with err: %v", proxyEnodeURLPair[1], err)
+			}
+		}
+
+		if !ctx.GlobalBool(NoDiscoverFlag.Name) {
+			Fatalf("Option --%s must be used if option --%s is used", NoDiscoverFlag.Name, ProxiedFlag.Name)
+		}
+	}
+}
+
+// checkExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
 func checkExclusive(ctx *cli.Context, args ...interface{}) {

consensus/consensus.go

@@ -18,7 +18,6 @@
 package consensus
 
 import (
-	"crypto/ecdsa"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -132,14 +131,26 @@ type GenesisAccount interface {
 
 // Handler should be implemented if the consensus needs to handle and send peer messages
 type Handler interface {
+	// NewWork handles a new work event from the miner
+	NewWork() error
+
 	// NewChainHead handles a new head block
-	NewChainHead() error
+	NewChainHead(*types.Block)
 
 	// HandleMsg handles a message from peer
-	HandleMsg(address common.Address, data p2p.Msg) (bool, error)
+	HandleMsg(address common.Address, data p2p.Msg, peer Peer) (bool, error)
 
 	// SetBroadcaster sets the broadcaster to send message to peers
 	SetBroadcaster(Broadcaster)
+
+	// SetP2PServer sets the p2p server to connect/disconnect to/from peers
+	SetP2PServer(P2PServer)
+
+	// RegisterPeer will notify the consensus engine that a new peer has been added
+	RegisterPeer(peer Peer, fromProxiedNode bool)
+
+	// UnregisterPeer will notify the consensus engine that a new peer has been removed
+	UnregisterPeer(peer Peer, fromProxiedNode bool)
 }
 
 // PoW is a consensus engine based on proof-of-work.
@@ -154,8 +165,6 @@ type PoW interface {
 type Istanbul interface {
 	Engine
 
-	GetNodeKey() *ecdsa.PrivateKey
-
 	SetChain(chain ChainReader, currentBlock func() *types.Block)
 
 	// Start starts the engine

consensus/consensustest/mockprotocol.go

@@ -0,0 +1,49 @@
+// Copyright 2017 The Celo Authors
+// This file is part of the celo library.
+//
+// The celo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The celo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the celo library. If not, see <http://www.gnu.org/licenses/>.
+
+package consensustest
+
+import (
+	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/enode"
+)
+
+type MockBroadcaster struct{}
+
+func (b *MockBroadcaster) Enqueue(id string, block *types.Block) {
+}
+
+func (b *MockBroadcaster) FindPeers(targets map[enode.ID]bool, purpose p2p.PurposeFlag) map[enode.ID]consensus.Peer {
+	return make(map[enode.ID]consensus.Peer)
+}
+
+type MockP2PServer struct {
+	Node *enode.Node
+}
+
+func (serv *MockP2PServer) Self() *enode.Node {
+	return serv.Node
+}
+
+func (serv *MockP2PServer) AddPeer(node *enode.Node, purpose p2p.PurposeFlag) {}
+
+func (serv *MockP2PServer) RemovePeer(node *enode.Node, purpose p2p.PurposeFlag) {}
+
+func (serv *MockP2PServer) AddTrustedPeer(node *enode.Node, purpose p2p.PurposeFlag) {}
+
+func (serv *MockP2PServer) RemoveTrustedPeer(node *enode.Node, purpose p2p.PurposeFlag) {}

consensus/istanbul/backend.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/p2p/enode"
 )
 
 // SignerFn is a signer callback function to request a hash to be signed by a
@@ -40,20 +39,17 @@ type Backend interface {
 	// Address returns the owner's address
 	Address() common.Address
 
-	// Enode returns the owner's enode
-	Enode() *enode.Node
-
 	// Validators returns the validator set
 	Validators(proposal Proposal) ValidatorSet
 
 	// EventMux returns the event mux in backend
 	EventMux() *event.TypeMux
 
-	// Broadcast sends a message to all validators (include self)
-	Broadcast(valSet ValidatorSet, payload []byte) error
+	// BroadcastConsensusMsg sends a message to all validators (include self)
+	BroadcastConsensusMsg(validators []common.Address, payload []byte) error
 
 	// Gossip sends a message to all validators (exclude self)
-	Gossip(valSet ValidatorSet, payload []byte, msgCode uint64, ignoreCache bool) error
+	Gossip(validators []common.Address, payload []byte, ethMsgCode uint64, ignoreCache bool) error
 
 	// Commit delivers an approved proposal to backend.
 	// The delivered proposal will be put into blockchain.
@@ -86,7 +82,7 @@ type Backend interface {
 	// ParentValidators returns the validator set of the given proposal's parent block
 	ParentValidators(proposal Proposal) ValidatorSet
 
-	// RefreshValPeers will connect all all the validators in the valset and disconnect validator peers that are not in the set
+	// RefreshValPeers will connect with all the validators in the valset and disconnect validator peers that are not in the set
 	RefreshValPeers(valset ValidatorSet)
 
 	// Authorize injects a private key into the consensus engine.